2024-12-07 15:13:33,609 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-12-07 15:13:33,624 main DEBUG Took 0.013114 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-07 15:13:33,625 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-07 15:13:33,625 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-07 15:13:33,626 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-07 15:13:33,627 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 15:13:33,633 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-07 15:13:33,646 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 15:13:33,647 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 15:13:33,648 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 15:13:33,648 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 15:13:33,649 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 15:13:33,649 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 15:13:33,651 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 15:13:33,651 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 15:13:33,652 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 15:13:33,652 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 15:13:33,653 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 15:13:33,653 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 15:13:33,654 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 15:13:33,654 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 15:13:33,655 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 15:13:33,655 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 15:13:33,656 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 15:13:33,656 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 15:13:33,657 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 15:13:33,657 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 15:13:33,658 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 15:13:33,658 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 15:13:33,659 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 15:13:33,659 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 15:13:33,660 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 15:13:33,660 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-07 15:13:33,662 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 15:13:33,664 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-07 15:13:33,666 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-07 15:13:33,666 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-07 15:13:33,668 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-07 15:13:33,668 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-07 15:13:33,678 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-07 15:13:33,680 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-07 15:13:33,681 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-07 15:13:33,682 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-07 15:13:33,682 main DEBUG createAppenders(={Console}) 2024-12-07 15:13:33,682 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-12-07 15:13:33,683 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-12-07 15:13:33,683 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-12-07 15:13:33,683 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-07 15:13:33,684 main DEBUG OutputStream closed 2024-12-07 15:13:33,684 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-07 15:13:33,684 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-07 15:13:33,684 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-12-07 15:13:33,748 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-07 15:13:33,750 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-07 15:13:33,751 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-07 15:13:33,752 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-07 15:13:33,752 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-07 15:13:33,753 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-07 15:13:33,753 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-07 15:13:33,753 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-07 15:13:33,753 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-07 15:13:33,754 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-07 15:13:33,754 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-07 15:13:33,754 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-07 15:13:33,754 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-07 15:13:33,755 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-07 15:13:33,755 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-07 15:13:33,755 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-07 15:13:33,756 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-07 15:13:33,756 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-07 15:13:33,758 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-07 15:13:33,759 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-12-07 15:13:33,759 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-07 15:13:33,760 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-12-07T15:13:33,987 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5975031d-e054-e0fa-13a8-f8e344c3a20d 2024-12-07 15:13:33,990 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-07 15:13:33,990 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-07T15:13:33,998 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-12-07T15:13:34,016 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-07T15:13:34,030 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5975031d-e054-e0fa-13a8-f8e344c3a20d/cluster_8a1d19e0-6a6c-4b1f-8318-b9e148680dc0, deleteOnExit=true 2024-12-07T15:13:34,031 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-07T15:13:34,031 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5975031d-e054-e0fa-13a8-f8e344c3a20d/test.cache.data in system properties and HBase conf 2024-12-07T15:13:34,032 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5975031d-e054-e0fa-13a8-f8e344c3a20d/hadoop.tmp.dir in system properties and HBase conf 2024-12-07T15:13:34,033 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5975031d-e054-e0fa-13a8-f8e344c3a20d/hadoop.log.dir in system properties and HBase conf 2024-12-07T15:13:34,033 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5975031d-e054-e0fa-13a8-f8e344c3a20d/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-07T15:13:34,034 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5975031d-e054-e0fa-13a8-f8e344c3a20d/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-07T15:13:34,034 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-07T15:13:34,121 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-07T15:13:34,216 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-07T15:13:34,220 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5975031d-e054-e0fa-13a8-f8e344c3a20d/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-07T15:13:34,220 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5975031d-e054-e0fa-13a8-f8e344c3a20d/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-07T15:13:34,221 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5975031d-e054-e0fa-13a8-f8e344c3a20d/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-07T15:13:34,221 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5975031d-e054-e0fa-13a8-f8e344c3a20d/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-07T15:13:34,221 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5975031d-e054-e0fa-13a8-f8e344c3a20d/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-07T15:13:34,222 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5975031d-e054-e0fa-13a8-f8e344c3a20d/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-07T15:13:34,222 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5975031d-e054-e0fa-13a8-f8e344c3a20d/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-07T15:13:34,223 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5975031d-e054-e0fa-13a8-f8e344c3a20d/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-07T15:13:34,223 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5975031d-e054-e0fa-13a8-f8e344c3a20d/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-07T15:13:34,223 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5975031d-e054-e0fa-13a8-f8e344c3a20d/nfs.dump.dir in system properties and HBase conf 2024-12-07T15:13:34,224 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5975031d-e054-e0fa-13a8-f8e344c3a20d/java.io.tmpdir in system properties and HBase conf 2024-12-07T15:13:34,224 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5975031d-e054-e0fa-13a8-f8e344c3a20d/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-07T15:13:34,225 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5975031d-e054-e0fa-13a8-f8e344c3a20d/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-07T15:13:34,225 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5975031d-e054-e0fa-13a8-f8e344c3a20d/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-07T15:13:35,053 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-07T15:13:35,142 INFO [Time-limited test {}] log.Log(170): Logging initialized @2338ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-07T15:13:35,216 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-07T15:13:35,274 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-07T15:13:35,293 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-07T15:13:35,293 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-07T15:13:35,294 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-07T15:13:35,307 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-07T15:13:35,310 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5975031d-e054-e0fa-13a8-f8e344c3a20d/hadoop.log.dir/,AVAILABLE} 2024-12-07T15:13:35,312 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-07T15:13:35,540 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5975031d-e054-e0fa-13a8-f8e344c3a20d/java.io.tmpdir/jetty-localhost-42807-hadoop-hdfs-3_4_1-tests_jar-_-any-4051235239694372635/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-07T15:13:35,546 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:42807} 2024-12-07T15:13:35,547 INFO [Time-limited test {}] server.Server(415): Started @2743ms 2024-12-07T15:13:35,969 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-07T15:13:35,978 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-07T15:13:35,979 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-07T15:13:35,979 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-07T15:13:35,980 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-07T15:13:35,981 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5975031d-e054-e0fa-13a8-f8e344c3a20d/hadoop.log.dir/,AVAILABLE} 2024-12-07T15:13:35,982 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-07T15:13:36,105 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5975031d-e054-e0fa-13a8-f8e344c3a20d/java.io.tmpdir/jetty-localhost-37971-hadoop-hdfs-3_4_1-tests_jar-_-any-1411686271385848156/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-07T15:13:36,106 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:37971} 2024-12-07T15:13:36,106 INFO [Time-limited test {}] server.Server(415): Started @3303ms 2024-12-07T15:13:36,166 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-07T15:13:36,686 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5975031d-e054-e0fa-13a8-f8e344c3a20d/cluster_8a1d19e0-6a6c-4b1f-8318-b9e148680dc0/data/data2/current/BP-848825693-172.17.0.2-1733584414802/current, will proceed with Du for space computation calculation, 2024-12-07T15:13:36,686 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5975031d-e054-e0fa-13a8-f8e344c3a20d/cluster_8a1d19e0-6a6c-4b1f-8318-b9e148680dc0/data/data1/current/BP-848825693-172.17.0.2-1733584414802/current, will proceed with Du for space computation calculation, 2024-12-07T15:13:36,719 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-07T15:13:36,771 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x4f714e9f1b19a7cd with lease ID 0xc1e2418f8905d19f: Processing first storage report for DS-dfc42307-6e6f-46b5-8a11-4dfa61994a03 from datanode DatanodeRegistration(127.0.0.1:43901, datanodeUuid=4d6fefba-0d69-4d1c-92e9-8f4bfc1f4da9, infoPort=43455, infoSecurePort=0, ipcPort=33845, storageInfo=lv=-57;cid=testClusterID;nsid=107222529;c=1733584414802) 2024-12-07T15:13:36,772 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x4f714e9f1b19a7cd with lease ID 0xc1e2418f8905d19f: from storage DS-dfc42307-6e6f-46b5-8a11-4dfa61994a03 node DatanodeRegistration(127.0.0.1:43901, datanodeUuid=4d6fefba-0d69-4d1c-92e9-8f4bfc1f4da9, infoPort=43455, infoSecurePort=0, ipcPort=33845, storageInfo=lv=-57;cid=testClusterID;nsid=107222529;c=1733584414802), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-12-07T15:13:36,772 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x4f714e9f1b19a7cd with lease ID 0xc1e2418f8905d19f: Processing first storage report for DS-99e39312-e8c0-4ae1-a9b7-07d32ea42b14 from datanode DatanodeRegistration(127.0.0.1:43901, datanodeUuid=4d6fefba-0d69-4d1c-92e9-8f4bfc1f4da9, infoPort=43455, infoSecurePort=0, ipcPort=33845, storageInfo=lv=-57;cid=testClusterID;nsid=107222529;c=1733584414802) 2024-12-07T15:13:36,772 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x4f714e9f1b19a7cd with lease ID 0xc1e2418f8905d19f: from storage DS-99e39312-e8c0-4ae1-a9b7-07d32ea42b14 node DatanodeRegistration(127.0.0.1:43901, datanodeUuid=4d6fefba-0d69-4d1c-92e9-8f4bfc1f4da9, infoPort=43455, infoSecurePort=0, ipcPort=33845, storageInfo=lv=-57;cid=testClusterID;nsid=107222529;c=1733584414802), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-07T15:13:36,779 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5975031d-e054-e0fa-13a8-f8e344c3a20d 2024-12-07T15:13:36,860 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5975031d-e054-e0fa-13a8-f8e344c3a20d/cluster_8a1d19e0-6a6c-4b1f-8318-b9e148680dc0/zookeeper_0, clientPort=52406, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5975031d-e054-e0fa-13a8-f8e344c3a20d/cluster_8a1d19e0-6a6c-4b1f-8318-b9e148680dc0/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5975031d-e054-e0fa-13a8-f8e344c3a20d/cluster_8a1d19e0-6a6c-4b1f-8318-b9e148680dc0/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-07T15:13:36,871 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=52406 2024-12-07T15:13:36,884 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-07T15:13:36,888 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-07T15:13:37,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741825_1001 (size=7) 2024-12-07T15:13:37,550 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd with version=8 2024-12-07T15:13:37,551 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/hbase-staging 2024-12-07T15:13:37,652 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-07T15:13:37,897 INFO [Time-limited test {}] client.ConnectionUtils(128): master/d6797ebe9c52:0 server-side Connection retries=45 2024-12-07T15:13:37,908 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-07T15:13:37,908 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-07T15:13:37,913 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-07T15:13:37,913 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-07T15:13:37,913 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-07T15:13:38,054 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-07T15:13:38,126 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-07T15:13:38,137 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-07T15:13:38,142 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-07T15:13:38,172 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 3069 (auto-detected) 2024-12-07T15:13:38,173 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-07T15:13:38,191 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36915 2024-12-07T15:13:38,213 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:36915 connecting to ZooKeeper ensemble=127.0.0.1:52406 2024-12-07T15:13:38,248 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:369150x0, quorum=127.0.0.1:52406, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-07T15:13:38,251 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:36915-0x1018d7cdad70000 connected 2024-12-07T15:13:38,276 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-07T15:13:38,278 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-07T15:13:38,287 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36915-0x1018d7cdad70000, quorum=127.0.0.1:52406, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-07T15:13:38,291 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd, hbase.cluster.distributed=false 2024-12-07T15:13:38,315 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36915-0x1018d7cdad70000, quorum=127.0.0.1:52406, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-07T15:13:38,321 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36915 2024-12-07T15:13:38,321 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36915 2024-12-07T15:13:38,322 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36915 2024-12-07T15:13:38,322 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36915 2024-12-07T15:13:38,323 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36915 2024-12-07T15:13:38,441 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/d6797ebe9c52:0 server-side Connection retries=45 2024-12-07T15:13:38,443 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-07T15:13:38,443 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-07T15:13:38,444 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-07T15:13:38,444 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-07T15:13:38,444 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-07T15:13:38,447 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-07T15:13:38,450 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-07T15:13:38,451 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46009 2024-12-07T15:13:38,453 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46009 connecting to ZooKeeper ensemble=127.0.0.1:52406 2024-12-07T15:13:38,454 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-07T15:13:38,457 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-07T15:13:38,465 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:460090x0, quorum=127.0.0.1:52406, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-07T15:13:38,466 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46009-0x1018d7cdad70001, quorum=127.0.0.1:52406, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-07T15:13:38,465 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46009-0x1018d7cdad70001 connected 2024-12-07T15:13:38,470 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-07T15:13:38,478 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-07T15:13:38,480 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46009-0x1018d7cdad70001, quorum=127.0.0.1:52406, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-07T15:13:38,485 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46009-0x1018d7cdad70001, quorum=127.0.0.1:52406, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-07T15:13:38,486 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46009 2024-12-07T15:13:38,490 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46009 2024-12-07T15:13:38,491 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46009 2024-12-07T15:13:38,493 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46009 2024-12-07T15:13:38,493 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46009 2024-12-07T15:13:38,508 DEBUG [M:0;d6797ebe9c52:36915 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;d6797ebe9c52:36915 2024-12-07T15:13:38,509 INFO [master/d6797ebe9c52:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/d6797ebe9c52,36915,1733584417704 2024-12-07T15:13:38,516 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46009-0x1018d7cdad70001, quorum=127.0.0.1:52406, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-07T15:13:38,516 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36915-0x1018d7cdad70000, quorum=127.0.0.1:52406, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-07T15:13:38,518 DEBUG [master/d6797ebe9c52:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36915-0x1018d7cdad70000, quorum=127.0.0.1:52406, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/d6797ebe9c52,36915,1733584417704 2024-12-07T15:13:38,538 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46009-0x1018d7cdad70001, quorum=127.0.0.1:52406, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-07T15:13:38,538 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36915-0x1018d7cdad70000, quorum=127.0.0.1:52406, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T15:13:38,538 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46009-0x1018d7cdad70001, quorum=127.0.0.1:52406, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T15:13:38,539 DEBUG [master/d6797ebe9c52:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36915-0x1018d7cdad70000, quorum=127.0.0.1:52406, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-07T15:13:38,540 INFO [master/d6797ebe9c52:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/d6797ebe9c52,36915,1733584417704 from backup master directory 2024-12-07T15:13:38,544 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46009-0x1018d7cdad70001, quorum=127.0.0.1:52406, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-07T15:13:38,544 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36915-0x1018d7cdad70000, quorum=127.0.0.1:52406, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/d6797ebe9c52,36915,1733584417704 2024-12-07T15:13:38,544 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36915-0x1018d7cdad70000, quorum=127.0.0.1:52406, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-07T15:13:38,544 WARN [master/d6797ebe9c52:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-07T15:13:38,545 INFO [master/d6797ebe9c52:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=d6797ebe9c52,36915,1733584417704 2024-12-07T15:13:38,547 INFO [master/d6797ebe9c52:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-07T15:13:38,549 INFO [master/d6797ebe9c52:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-07T15:13:38,612 DEBUG [master/d6797ebe9c52:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/hbase.id] with ID: f8956274-ed33-4e16-82eb-d5d5911ec893 2024-12-07T15:13:38,613 DEBUG [master/d6797ebe9c52:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/.tmp/hbase.id 2024-12-07T15:13:38,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741826_1002 (size=42) 2024-12-07T15:13:38,627 DEBUG [master/d6797ebe9c52:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/.tmp/hbase.id]:[hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/hbase.id] 2024-12-07T15:13:38,675 INFO [master/d6797ebe9c52:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-07T15:13:38,680 INFO [master/d6797ebe9c52:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-07T15:13:38,702 INFO [master/d6797ebe9c52:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 19ms. 2024-12-07T15:13:38,706 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36915-0x1018d7cdad70000, quorum=127.0.0.1:52406, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T15:13:38,706 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46009-0x1018d7cdad70001, quorum=127.0.0.1:52406, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T15:13:38,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741827_1003 (size=196) 2024-12-07T15:13:38,745 INFO [master/d6797ebe9c52:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-07T15:13:38,748 INFO [master/d6797ebe9c52:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-07T15:13:38,771 DEBUG [master/d6797ebe9c52:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:13:38,777 INFO [master/d6797ebe9c52:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-07T15:13:38,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741828_1004 (size=1189) 2024-12-07T15:13:38,843 INFO [master/d6797ebe9c52:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/MasterData/data/master/store 2024-12-07T15:13:38,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741829_1005 (size=34) 2024-12-07T15:13:39,281 INFO [master/d6797ebe9c52:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-07T15:13:39,284 DEBUG [master/d6797ebe9c52:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T15:13:39,285 DEBUG [master/d6797ebe9c52:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-07T15:13:39,285 INFO [master/d6797ebe9c52:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-07T15:13:39,286 DEBUG [master/d6797ebe9c52:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-07T15:13:39,287 DEBUG [master/d6797ebe9c52:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-07T15:13:39,287 DEBUG [master/d6797ebe9c52:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-07T15:13:39,287 INFO [master/d6797ebe9c52:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-07T15:13:39,288 DEBUG [master/d6797ebe9c52:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733584419285Disabling compacts and flushes for region at 1733584419285Disabling writes for close at 1733584419287 (+2 ms)Writing region close event to WAL at 1733584419287Closed at 1733584419287 2024-12-07T15:13:39,291 WARN [master/d6797ebe9c52:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/MasterData/data/master/store/.initializing 2024-12-07T15:13:39,291 DEBUG [master/d6797ebe9c52:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/MasterData/WALs/d6797ebe9c52,36915,1733584417704 2024-12-07T15:13:39,299 INFO [master/d6797ebe9c52:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-07T15:13:39,313 INFO [master/d6797ebe9c52:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d6797ebe9c52%2C36915%2C1733584417704, suffix=, logDir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/MasterData/WALs/d6797ebe9c52,36915,1733584417704, archiveDir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/MasterData/oldWALs, maxLogs=10 2024-12-07T15:13:39,336 DEBUG [master/d6797ebe9c52:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/MasterData/WALs/d6797ebe9c52,36915,1733584417704/d6797ebe9c52%2C36915%2C1733584417704.1733584419318, exclude list is [], retry=0 2024-12-07T15:13:39,353 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43901,DS-dfc42307-6e6f-46b5-8a11-4dfa61994a03,DISK] 2024-12-07T15:13:39,356 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-07T15:13:39,398 INFO [master/d6797ebe9c52:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/MasterData/WALs/d6797ebe9c52,36915,1733584417704/d6797ebe9c52%2C36915%2C1733584417704.1733584419318 2024-12-07T15:13:39,399 DEBUG [master/d6797ebe9c52:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43455:43455)] 2024-12-07T15:13:39,400 DEBUG [master/d6797ebe9c52:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-07T15:13:39,400 DEBUG [master/d6797ebe9c52:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T15:13:39,404 DEBUG [master/d6797ebe9c52:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-07T15:13:39,405 DEBUG [master/d6797ebe9c52:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-07T15:13:39,442 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-07T15:13:39,468 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-07T15:13:39,471 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:13:39,473 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T15:13:39,474 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-07T15:13:39,476 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-07T15:13:39,477 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:13:39,478 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T15:13:39,478 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-07T15:13:39,480 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-07T15:13:39,481 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:13:39,481 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T15:13:39,482 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-07T15:13:39,484 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-07T15:13:39,484 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:13:39,486 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T15:13:39,486 DEBUG [master/d6797ebe9c52:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-07T15:13:39,490 DEBUG [master/d6797ebe9c52:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-07T15:13:39,491 DEBUG [master/d6797ebe9c52:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-07T15:13:39,496 DEBUG [master/d6797ebe9c52:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-07T15:13:39,497 DEBUG [master/d6797ebe9c52:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-07T15:13:39,500 DEBUG [master/d6797ebe9c52:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-07T15:13:39,504 DEBUG [master/d6797ebe9c52:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-07T15:13:39,509 DEBUG [master/d6797ebe9c52:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T15:13:39,510 INFO [master/d6797ebe9c52:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65192835, jitterRate=-0.02855105698108673}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-07T15:13:39,516 DEBUG [master/d6797ebe9c52:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733584419417Initializing all the Stores at 1733584419419 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733584419420 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733584419421 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733584419421Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733584419421Cleaning up temporary data from old regions at 1733584419497 (+76 ms)Region opened successfully at 1733584419516 (+19 ms) 2024-12-07T15:13:39,517 INFO [master/d6797ebe9c52:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-07T15:13:39,551 DEBUG [master/d6797ebe9c52:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39e73a9f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d6797ebe9c52/172.17.0.2:0 2024-12-07T15:13:39,584 INFO [master/d6797ebe9c52:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-07T15:13:39,595 INFO [master/d6797ebe9c52:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-07T15:13:39,595 INFO [master/d6797ebe9c52:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-07T15:13:39,598 INFO [master/d6797ebe9c52:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-07T15:13:39,600 INFO [master/d6797ebe9c52:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-07T15:13:39,605 INFO [master/d6797ebe9c52:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-12-07T15:13:39,605 INFO [master/d6797ebe9c52:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-07T15:13:39,629 INFO [master/d6797ebe9c52:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-07T15:13:39,638 DEBUG [master/d6797ebe9c52:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36915-0x1018d7cdad70000, quorum=127.0.0.1:52406, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-07T15:13:39,640 DEBUG [master/d6797ebe9c52:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-07T15:13:39,642 INFO [master/d6797ebe9c52:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-07T15:13:39,643 DEBUG [master/d6797ebe9c52:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36915-0x1018d7cdad70000, quorum=127.0.0.1:52406, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-07T15:13:39,645 DEBUG [master/d6797ebe9c52:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-07T15:13:39,647 INFO [master/d6797ebe9c52:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-07T15:13:39,650 DEBUG [master/d6797ebe9c52:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36915-0x1018d7cdad70000, quorum=127.0.0.1:52406, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-07T15:13:39,652 DEBUG [master/d6797ebe9c52:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-07T15:13:39,654 DEBUG [master/d6797ebe9c52:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36915-0x1018d7cdad70000, quorum=127.0.0.1:52406, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-07T15:13:39,655 DEBUG [master/d6797ebe9c52:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-07T15:13:39,672 DEBUG [master/d6797ebe9c52:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36915-0x1018d7cdad70000, quorum=127.0.0.1:52406, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-07T15:13:39,673 DEBUG [master/d6797ebe9c52:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-07T15:13:39,677 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36915-0x1018d7cdad70000, quorum=127.0.0.1:52406, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-07T15:13:39,677 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46009-0x1018d7cdad70001, quorum=127.0.0.1:52406, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-07T15:13:39,677 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36915-0x1018d7cdad70000, quorum=127.0.0.1:52406, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T15:13:39,677 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46009-0x1018d7cdad70001, quorum=127.0.0.1:52406, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T15:13:39,680 INFO [master/d6797ebe9c52:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=d6797ebe9c52,36915,1733584417704, sessionid=0x1018d7cdad70000, setting cluster-up flag (Was=false) 2024-12-07T15:13:39,695 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46009-0x1018d7cdad70001, quorum=127.0.0.1:52406, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T15:13:39,695 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36915-0x1018d7cdad70000, quorum=127.0.0.1:52406, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T15:13:39,701 DEBUG [master/d6797ebe9c52:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-07T15:13:39,703 DEBUG [master/d6797ebe9c52:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d6797ebe9c52,36915,1733584417704 2024-12-07T15:13:39,708 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36915-0x1018d7cdad70000, quorum=127.0.0.1:52406, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T15:13:39,708 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46009-0x1018d7cdad70001, quorum=127.0.0.1:52406, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T15:13:39,714 DEBUG [master/d6797ebe9c52:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-07T15:13:39,715 DEBUG [master/d6797ebe9c52:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d6797ebe9c52,36915,1733584417704 2024-12-07T15:13:39,721 INFO [master/d6797ebe9c52:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-07T15:13:39,797 INFO [RS:0;d6797ebe9c52:46009 {}] regionserver.HRegionServer(746): ClusterId : f8956274-ed33-4e16-82eb-d5d5911ec893 2024-12-07T15:13:39,798 DEBUG [master/d6797ebe9c52:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-07T15:13:39,801 DEBUG [RS:0;d6797ebe9c52:46009 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-07T15:13:39,806 DEBUG [RS:0;d6797ebe9c52:46009 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-07T15:13:39,807 DEBUG [RS:0;d6797ebe9c52:46009 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-07T15:13:39,810 INFO [master/d6797ebe9c52:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-07T15:13:39,810 DEBUG [RS:0;d6797ebe9c52:46009 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-07T15:13:39,810 DEBUG [RS:0;d6797ebe9c52:46009 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ea12802, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d6797ebe9c52/172.17.0.2:0 2024-12-07T15:13:39,818 INFO [master/d6797ebe9c52:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-07T15:13:39,826 DEBUG [master/d6797ebe9c52:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: d6797ebe9c52,36915,1733584417704 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-07T15:13:39,832 DEBUG [RS:0;d6797ebe9c52:46009 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;d6797ebe9c52:46009 2024-12-07T15:13:39,835 DEBUG [master/d6797ebe9c52:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/d6797ebe9c52:0, corePoolSize=5, maxPoolSize=5 2024-12-07T15:13:39,835 DEBUG [master/d6797ebe9c52:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/d6797ebe9c52:0, corePoolSize=5, maxPoolSize=5 2024-12-07T15:13:39,835 DEBUG [master/d6797ebe9c52:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/d6797ebe9c52:0, corePoolSize=5, maxPoolSize=5 2024-12-07T15:13:39,835 DEBUG [master/d6797ebe9c52:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/d6797ebe9c52:0, corePoolSize=5, maxPoolSize=5 2024-12-07T15:13:39,836 DEBUG [master/d6797ebe9c52:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/d6797ebe9c52:0, corePoolSize=10, maxPoolSize=10 2024-12-07T15:13:39,836 DEBUG [master/d6797ebe9c52:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/d6797ebe9c52:0, corePoolSize=1, maxPoolSize=1 2024-12-07T15:13:39,836 DEBUG [master/d6797ebe9c52:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/d6797ebe9c52:0, corePoolSize=2, maxPoolSize=2 2024-12-07T15:13:39,837 DEBUG [master/d6797ebe9c52:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/d6797ebe9c52:0, corePoolSize=1, maxPoolSize=1 2024-12-07T15:13:39,837 INFO [RS:0;d6797ebe9c52:46009 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-07T15:13:39,837 INFO [RS:0;d6797ebe9c52:46009 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-07T15:13:39,837 DEBUG [RS:0;d6797ebe9c52:46009 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-07T15:13:39,840 INFO [RS:0;d6797ebe9c52:46009 {}] regionserver.HRegionServer(2659): reportForDuty to master=d6797ebe9c52,36915,1733584417704 with port=46009, startcode=1733584418401 2024-12-07T15:13:39,854 DEBUG [RS:0;d6797ebe9c52:46009 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-07T15:13:39,854 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-07T15:13:39,855 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-07T15:13:39,861 INFO [master/d6797ebe9c52:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733584449861 2024-12-07T15:13:39,863 INFO [master/d6797ebe9c52:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-07T15:13:39,863 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:13:39,863 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-07T15:13:39,864 INFO [master/d6797ebe9c52:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-07T15:13:39,869 INFO [master/d6797ebe9c52:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-07T15:13:39,869 INFO [master/d6797ebe9c52:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-07T15:13:39,870 INFO [master/d6797ebe9c52:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-07T15:13:39,870 INFO [master/d6797ebe9c52:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-07T15:13:39,877 INFO [master/d6797ebe9c52:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-07T15:13:39,883 INFO [master/d6797ebe9c52:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-07T15:13:39,885 INFO [master/d6797ebe9c52:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-07T15:13:39,885 INFO [master/d6797ebe9c52:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-07T15:13:39,891 INFO [master/d6797ebe9c52:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-07T15:13:39,891 INFO [master/d6797ebe9c52:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-07T15:13:39,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741831_1007 (size=1321) 2024-12-07T15:13:39,895 DEBUG [master/d6797ebe9c52:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/d6797ebe9c52:0:becomeActiveMaster-HFileCleaner.large.0-1733584419894,5,FailOnTimeoutGroup] 2024-12-07T15:13:39,896 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-07T15:13:39,897 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40405, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-07T15:13:39,897 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd 2024-12-07T15:13:39,904 DEBUG [master/d6797ebe9c52:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/d6797ebe9c52:0:becomeActiveMaster-HFileCleaner.small.0-1733584419898,5,FailOnTimeoutGroup] 2024-12-07T15:13:39,904 INFO [master/d6797ebe9c52:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-07T15:13:39,905 INFO [master/d6797ebe9c52:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-07T15:13:39,905 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36915 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:39,906 INFO [master/d6797ebe9c52:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-07T15:13:39,907 INFO [master/d6797ebe9c52:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-07T15:13:39,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741832_1008 (size=32) 2024-12-07T15:13:39,923 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T15:13:39,930 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-07T15:13:39,934 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-07T15:13:39,935 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:13:39,936 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T15:13:39,936 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-07T15:13:39,939 DEBUG [RS:0;d6797ebe9c52:46009 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-12-07T15:13:39,939 WARN [RS:0;d6797ebe9c52:46009 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-12-07T15:13:39,943 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-07T15:13:39,943 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:13:39,944 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T15:13:39,945 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-07T15:13:39,948 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-07T15:13:39,948 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:13:39,950 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T15:13:39,950 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-07T15:13:39,953 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-07T15:13:39,953 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:13:39,955 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T15:13:39,956 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-07T15:13:39,958 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/hbase/meta/1588230740 2024-12-07T15:13:39,959 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/hbase/meta/1588230740 2024-12-07T15:13:39,962 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-07T15:13:39,962 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-07T15:13:39,963 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-07T15:13:39,967 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-07T15:13:39,974 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T15:13:39,975 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65784108, jitterRate=-0.019740402698516846}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-07T15:13:39,979 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733584419924Initializing all the Stores at 1733584419926 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733584419926Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733584419928 (+2 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733584419928Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733584419928Cleaning up temporary data from old regions at 1733584419962 (+34 ms)Region opened successfully at 1733584419979 (+17 ms) 2024-12-07T15:13:39,979 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-07T15:13:39,980 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-07T15:13:39,980 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-07T15:13:39,980 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-07T15:13:39,980 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-07T15:13:39,982 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-07T15:13:39,982 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733584419979Disabling compacts and flushes for region at 1733584419979Disabling writes for close at 1733584419980 (+1 ms)Writing region close event to WAL at 1733584419981 (+1 ms)Closed at 1733584419981 2024-12-07T15:13:39,990 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-07T15:13:39,990 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-07T15:13:39,999 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-07T15:13:40,009 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-07T15:13:40,013 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-07T15:13:40,040 INFO [RS:0;d6797ebe9c52:46009 {}] regionserver.HRegionServer(2659): reportForDuty to master=d6797ebe9c52,36915,1733584417704 with port=46009, startcode=1733584418401 2024-12-07T15:13:40,042 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36915 {}] master.ServerManager(363): Checking decommissioned status of RegionServer d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:40,045 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36915 {}] master.ServerManager(517): Registering regionserver=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:40,053 DEBUG [RS:0;d6797ebe9c52:46009 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd 2024-12-07T15:13:40,054 DEBUG [RS:0;d6797ebe9c52:46009 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:36357 2024-12-07T15:13:40,054 DEBUG [RS:0;d6797ebe9c52:46009 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-07T15:13:40,059 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36915-0x1018d7cdad70000, quorum=127.0.0.1:52406, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-07T15:13:40,061 DEBUG [RS:0;d6797ebe9c52:46009 {}] zookeeper.ZKUtil(111): regionserver:46009-0x1018d7cdad70001, quorum=127.0.0.1:52406, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:40,061 WARN [RS:0;d6797ebe9c52:46009 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-07T15:13:40,061 INFO [RS:0;d6797ebe9c52:46009 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-07T15:13:40,061 DEBUG [RS:0;d6797ebe9c52:46009 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/WALs/d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:40,063 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [d6797ebe9c52,46009,1733584418401] 2024-12-07T15:13:40,089 INFO [RS:0;d6797ebe9c52:46009 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-07T15:13:40,104 INFO [RS:0;d6797ebe9c52:46009 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-07T15:13:40,110 INFO [RS:0;d6797ebe9c52:46009 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-07T15:13:40,110 INFO [RS:0;d6797ebe9c52:46009 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T15:13:40,111 INFO [RS:0;d6797ebe9c52:46009 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-07T15:13:40,118 INFO [RS:0;d6797ebe9c52:46009 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-07T15:13:40,119 INFO [RS:0;d6797ebe9c52:46009 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-07T15:13:40,120 DEBUG [RS:0;d6797ebe9c52:46009 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/d6797ebe9c52:0, corePoolSize=1, maxPoolSize=1 2024-12-07T15:13:40,120 DEBUG [RS:0;d6797ebe9c52:46009 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/d6797ebe9c52:0, corePoolSize=1, maxPoolSize=1 2024-12-07T15:13:40,120 DEBUG [RS:0;d6797ebe9c52:46009 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/d6797ebe9c52:0, corePoolSize=1, maxPoolSize=1 2024-12-07T15:13:40,120 DEBUG [RS:0;d6797ebe9c52:46009 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/d6797ebe9c52:0, corePoolSize=1, maxPoolSize=1 2024-12-07T15:13:40,120 DEBUG [RS:0;d6797ebe9c52:46009 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/d6797ebe9c52:0, corePoolSize=1, maxPoolSize=1 2024-12-07T15:13:40,121 DEBUG [RS:0;d6797ebe9c52:46009 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/d6797ebe9c52:0, corePoolSize=2, maxPoolSize=2 2024-12-07T15:13:40,121 DEBUG [RS:0;d6797ebe9c52:46009 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/d6797ebe9c52:0, corePoolSize=1, maxPoolSize=1 2024-12-07T15:13:40,121 DEBUG [RS:0;d6797ebe9c52:46009 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/d6797ebe9c52:0, corePoolSize=1, maxPoolSize=1 2024-12-07T15:13:40,121 DEBUG [RS:0;d6797ebe9c52:46009 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/d6797ebe9c52:0, corePoolSize=1, maxPoolSize=1 2024-12-07T15:13:40,121 DEBUG [RS:0;d6797ebe9c52:46009 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/d6797ebe9c52:0, corePoolSize=1, maxPoolSize=1 2024-12-07T15:13:40,121 DEBUG [RS:0;d6797ebe9c52:46009 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/d6797ebe9c52:0, corePoolSize=1, maxPoolSize=1 2024-12-07T15:13:40,121 DEBUG [RS:0;d6797ebe9c52:46009 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/d6797ebe9c52:0, corePoolSize=1, maxPoolSize=1 2024-12-07T15:13:40,122 DEBUG [RS:0;d6797ebe9c52:46009 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/d6797ebe9c52:0, corePoolSize=3, maxPoolSize=3 2024-12-07T15:13:40,122 DEBUG [RS:0;d6797ebe9c52:46009 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0, corePoolSize=3, maxPoolSize=3 2024-12-07T15:13:40,123 INFO [RS:0;d6797ebe9c52:46009 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-07T15:13:40,123 INFO [RS:0;d6797ebe9c52:46009 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-07T15:13:40,124 INFO [RS:0;d6797ebe9c52:46009 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T15:13:40,124 INFO [RS:0;d6797ebe9c52:46009 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-07T15:13:40,124 INFO [RS:0;d6797ebe9c52:46009 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-07T15:13:40,124 INFO [RS:0;d6797ebe9c52:46009 {}] hbase.ChoreService(168): Chore ScheduledChore name=d6797ebe9c52,46009,1733584418401-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-07T15:13:40,148 INFO [RS:0;d6797ebe9c52:46009 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-07T15:13:40,150 INFO [RS:0;d6797ebe9c52:46009 {}] hbase.ChoreService(168): Chore ScheduledChore name=d6797ebe9c52,46009,1733584418401-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T15:13:40,150 INFO [RS:0;d6797ebe9c52:46009 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T15:13:40,150 INFO [RS:0;d6797ebe9c52:46009 {}] regionserver.Replication(171): d6797ebe9c52,46009,1733584418401 started 2024-12-07T15:13:40,164 WARN [d6797ebe9c52:36915 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-07T15:13:40,172 INFO [RS:0;d6797ebe9c52:46009 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T15:13:40,172 INFO [RS:0;d6797ebe9c52:46009 {}] regionserver.HRegionServer(1482): Serving as d6797ebe9c52,46009,1733584418401, RpcServer on d6797ebe9c52/172.17.0.2:46009, sessionid=0x1018d7cdad70001 2024-12-07T15:13:40,173 DEBUG [RS:0;d6797ebe9c52:46009 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-07T15:13:40,173 DEBUG [RS:0;d6797ebe9c52:46009 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:40,174 DEBUG [RS:0;d6797ebe9c52:46009 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd6797ebe9c52,46009,1733584418401' 2024-12-07T15:13:40,174 DEBUG [RS:0;d6797ebe9c52:46009 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-07T15:13:40,175 DEBUG [RS:0;d6797ebe9c52:46009 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-07T15:13:40,176 DEBUG [RS:0;d6797ebe9c52:46009 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-07T15:13:40,176 DEBUG [RS:0;d6797ebe9c52:46009 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-07T15:13:40,176 DEBUG [RS:0;d6797ebe9c52:46009 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:40,176 DEBUG [RS:0;d6797ebe9c52:46009 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd6797ebe9c52,46009,1733584418401' 2024-12-07T15:13:40,177 DEBUG [RS:0;d6797ebe9c52:46009 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-07T15:13:40,177 DEBUG [RS:0;d6797ebe9c52:46009 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-07T15:13:40,178 DEBUG [RS:0;d6797ebe9c52:46009 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-07T15:13:40,178 INFO [RS:0;d6797ebe9c52:46009 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-07T15:13:40,178 INFO [RS:0;d6797ebe9c52:46009 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-07T15:13:40,284 INFO [RS:0;d6797ebe9c52:46009 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-07T15:13:40,287 INFO [RS:0;d6797ebe9c52:46009 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d6797ebe9c52%2C46009%2C1733584418401, suffix=, logDir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/WALs/d6797ebe9c52,46009,1733584418401, archiveDir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/oldWALs, maxLogs=32 2024-12-07T15:13:40,306 DEBUG [RS:0;d6797ebe9c52:46009 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/WALs/d6797ebe9c52,46009,1733584418401/d6797ebe9c52%2C46009%2C1733584418401.1733584420290, exclude list is [], retry=0 2024-12-07T15:13:40,312 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43901,DS-dfc42307-6e6f-46b5-8a11-4dfa61994a03,DISK] 2024-12-07T15:13:40,316 INFO [RS:0;d6797ebe9c52:46009 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/WALs/d6797ebe9c52,46009,1733584418401/d6797ebe9c52%2C46009%2C1733584418401.1733584420290 2024-12-07T15:13:40,316 DEBUG [RS:0;d6797ebe9c52:46009 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43455:43455)] 2024-12-07T15:13:40,417 DEBUG [d6797ebe9c52:36915 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-07T15:13:40,429 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:40,436 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d6797ebe9c52,46009,1733584418401, state=OPENING 2024-12-07T15:13:40,441 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-07T15:13:40,446 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36915-0x1018d7cdad70000, quorum=127.0.0.1:52406, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T15:13:40,448 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T15:13:40,448 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46009-0x1018d7cdad70001, quorum=127.0.0.1:52406, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T15:13:40,449 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-07T15:13:40,449 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T15:13:40,452 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=d6797ebe9c52,46009,1733584418401}] 2024-12-07T15:13:40,635 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-07T15:13:40,640 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33061, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-07T15:13:40,654 INFO [RS_OPEN_META-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-07T15:13:40,654 INFO [RS_OPEN_META-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-07T15:13:40,655 INFO [RS_OPEN_META-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-07T15:13:40,659 INFO [RS_OPEN_META-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d6797ebe9c52%2C46009%2C1733584418401.meta, suffix=.meta, logDir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/WALs/d6797ebe9c52,46009,1733584418401, archiveDir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/oldWALs, maxLogs=32 2024-12-07T15:13:40,682 DEBUG [RS_OPEN_META-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/WALs/d6797ebe9c52,46009,1733584418401/d6797ebe9c52%2C46009%2C1733584418401.meta.1733584420661.meta, exclude list is [], retry=0 2024-12-07T15:13:40,688 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43901,DS-dfc42307-6e6f-46b5-8a11-4dfa61994a03,DISK] 2024-12-07T15:13:40,697 INFO [RS_OPEN_META-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/WALs/d6797ebe9c52,46009,1733584418401/d6797ebe9c52%2C46009%2C1733584418401.meta.1733584420661.meta 2024-12-07T15:13:40,700 DEBUG [RS_OPEN_META-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43455:43455)] 2024-12-07T15:13:40,701 DEBUG [RS_OPEN_META-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-07T15:13:40,703 DEBUG [RS_OPEN_META-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-07T15:13:40,707 DEBUG [RS_OPEN_META-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-07T15:13:40,714 INFO [RS_OPEN_META-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-07T15:13:40,720 DEBUG [RS_OPEN_META-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-07T15:13:40,721 DEBUG [RS_OPEN_META-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T15:13:40,721 DEBUG [RS_OPEN_META-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-07T15:13:40,722 DEBUG [RS_OPEN_META-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-07T15:13:40,729 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-07T15:13:40,731 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-07T15:13:40,732 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:13:40,733 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T15:13:40,734 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-07T15:13:40,736 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-07T15:13:40,736 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:13:40,737 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T15:13:40,737 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-07T15:13:40,739 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-07T15:13:40,740 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:13:40,741 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T15:13:40,742 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-07T15:13:40,744 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-07T15:13:40,744 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:13:40,745 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T15:13:40,745 DEBUG [RS_OPEN_META-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-07T15:13:40,747 DEBUG [RS_OPEN_META-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/hbase/meta/1588230740 2024-12-07T15:13:40,751 DEBUG [RS_OPEN_META-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/hbase/meta/1588230740 2024-12-07T15:13:40,754 DEBUG [RS_OPEN_META-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-07T15:13:40,754 DEBUG [RS_OPEN_META-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-07T15:13:40,755 DEBUG [RS_OPEN_META-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-07T15:13:40,759 DEBUG [RS_OPEN_META-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-07T15:13:40,761 INFO [RS_OPEN_META-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65722945, jitterRate=-0.02065180242061615}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-07T15:13:40,761 DEBUG [RS_OPEN_META-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-07T15:13:40,763 DEBUG [RS_OPEN_META-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733584420722Writing region info on filesystem at 1733584420723 (+1 ms)Initializing all the Stores at 1733584420725 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733584420725Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733584420728 (+3 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733584420728Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733584420728Cleaning up temporary data from old regions at 1733584420754 (+26 ms)Running coprocessor post-open hooks at 1733584420761 (+7 ms)Region opened successfully at 1733584420763 (+2 ms) 2024-12-07T15:13:40,772 INFO [RS_OPEN_META-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733584420624 2024-12-07T15:13:40,786 DEBUG [RS_OPEN_META-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-07T15:13:40,786 INFO [RS_OPEN_META-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-07T15:13:40,788 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:40,791 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d6797ebe9c52,46009,1733584418401, state=OPEN 2024-12-07T15:13:40,799 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46009-0x1018d7cdad70001, quorum=127.0.0.1:52406, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T15:13:40,799 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36915-0x1018d7cdad70000, quorum=127.0.0.1:52406, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T15:13:40,800 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T15:13:40,800 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T15:13:40,800 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:40,807 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-07T15:13:40,807 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d6797ebe9c52,46009,1733584418401 in 349 msec 2024-12-07T15:13:40,818 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-07T15:13:40,819 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 810 msec 2024-12-07T15:13:40,821 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-07T15:13:40,821 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-07T15:13:40,852 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T15:13:40,854 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6797ebe9c52,46009,1733584418401, seqNum=-1] 2024-12-07T15:13:40,888 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T15:13:40,893 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41181, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T15:13:40,924 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1720 sec 2024-12-07T15:13:40,924 INFO [master/d6797ebe9c52:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733584420924, completionTime=-1 2024-12-07T15:13:40,928 INFO [master/d6797ebe9c52:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-07T15:13:40,929 DEBUG [master/d6797ebe9c52:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-07T15:13:40,967 INFO [master/d6797ebe9c52:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-12-07T15:13:40,968 INFO [master/d6797ebe9c52:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733584480968 2024-12-07T15:13:40,968 INFO [master/d6797ebe9c52:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733584540968 2024-12-07T15:13:40,968 INFO [master/d6797ebe9c52:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 39 msec 2024-12-07T15:13:40,971 INFO [master/d6797ebe9c52:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d6797ebe9c52,36915,1733584417704-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T15:13:40,971 INFO [master/d6797ebe9c52:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d6797ebe9c52,36915,1733584417704-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T15:13:40,971 INFO [master/d6797ebe9c52:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d6797ebe9c52,36915,1733584417704-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T15:13:40,973 INFO [master/d6797ebe9c52:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-d6797ebe9c52:36915, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T15:13:40,974 INFO [master/d6797ebe9c52:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-07T15:13:40,975 INFO [master/d6797ebe9c52:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-07T15:13:40,983 DEBUG [master/d6797ebe9c52:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-07T15:13:41,075 INFO [master/d6797ebe9c52:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.530sec 2024-12-07T15:13:41,077 INFO [master/d6797ebe9c52:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-07T15:13:41,079 INFO [master/d6797ebe9c52:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-07T15:13:41,080 INFO [master/d6797ebe9c52:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-07T15:13:41,081 INFO [master/d6797ebe9c52:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-07T15:13:41,081 INFO [master/d6797ebe9c52:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-07T15:13:41,082 INFO [master/d6797ebe9c52:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d6797ebe9c52,36915,1733584417704-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-07T15:13:41,083 INFO [master/d6797ebe9c52:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d6797ebe9c52,36915,1733584417704-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-07T15:13:41,105 DEBUG [master/d6797ebe9c52:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-07T15:13:41,107 INFO [master/d6797ebe9c52:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-07T15:13:41,108 INFO [master/d6797ebe9c52:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d6797ebe9c52,36915,1733584417704-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T15:13:41,116 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45544f78, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:13:41,118 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-07T15:13:41,119 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-07T15:13:41,123 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6797ebe9c52,36915,-1 for getting cluster id 2024-12-07T15:13:41,127 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T15:13:41,137 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f8956274-ed33-4e16-82eb-d5d5911ec893' 2024-12-07T15:13:41,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T15:13:41,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f8956274-ed33-4e16-82eb-d5d5911ec893" 2024-12-07T15:13:41,145 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@519651b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:13:41,145 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6797ebe9c52,36915,-1] 2024-12-07T15:13:41,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T15:13:41,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:13:41,154 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49904, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T15:13:41,158 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2de67725, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:13:41,159 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T15:13:41,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6797ebe9c52,46009,1733584418401, seqNum=-1] 2024-12-07T15:13:41,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T15:13:41,176 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36006, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T15:13:41,200 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=d6797ebe9c52,36915,1733584417704 2024-12-07T15:13:41,216 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=255, ProcessCount=11, AvailableMemoryMB=7923 2024-12-07T15:13:41,228 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-07T15:13:41,233 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is d6797ebe9c52,36915,1733584417704 2024-12-07T15:13:41,236 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@a1209f1 2024-12-07T15:13:41,237 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-07T15:13:41,243 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49920, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-07T15:13:41,246 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-07T15:13:41,252 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-07T15:13:41,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-07T15:13:41,268 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-07T15:13:41,271 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-12-07T15:13:41,271 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:13:41,274 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-07T15:13:41,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-07T15:13:41,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741835_1011 (size=1061) 2024-12-07T15:13:41,304 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd 2024-12-07T15:13:41,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741836_1012 (size=53) 2024-12-07T15:13:41,324 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T15:13:41,325 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 53401f0681ce510d042c1599ea9bc7e2, disabling compactions & flushes 2024-12-07T15:13:41,325 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:41,325 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:41,325 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. after waiting 0 ms 2024-12-07T15:13:41,325 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:41,325 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:41,325 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 53401f0681ce510d042c1599ea9bc7e2: Waiting for close lock at 1733584421324Disabling compacts and flushes for region at 1733584421324Disabling writes for close at 1733584421325 (+1 ms)Writing region close event to WAL at 1733584421325Closed at 1733584421325 2024-12-07T15:13:41,328 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-07T15:13:41,335 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733584421329"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733584421329"}]},"ts":"1733584421329"} 2024-12-07T15:13:41,341 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-07T15:13:41,343 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-07T15:13:41,348 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733584421343"}]},"ts":"1733584421343"} 2024-12-07T15:13:41,354 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-07T15:13:41,356 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=53401f0681ce510d042c1599ea9bc7e2, ASSIGN}] 2024-12-07T15:13:41,359 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=53401f0681ce510d042c1599ea9bc7e2, ASSIGN 2024-12-07T15:13:41,361 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=53401f0681ce510d042c1599ea9bc7e2, ASSIGN; state=OFFLINE, location=d6797ebe9c52,46009,1733584418401; forceNewPlan=false, retain=false 2024-12-07T15:13:41,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-07T15:13:41,513 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=53401f0681ce510d042c1599ea9bc7e2, regionState=OPENING, regionLocation=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:41,517 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=53401f0681ce510d042c1599ea9bc7e2, ASSIGN because future has completed 2024-12-07T15:13:41,521 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401}] 2024-12-07T15:13:41,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-07T15:13:41,682 INFO [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:41,683 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} 2024-12-07T15:13:41,683 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 53401f0681ce510d042c1599ea9bc7e2 2024-12-07T15:13:41,684 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T15:13:41,684 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 53401f0681ce510d042c1599ea9bc7e2 2024-12-07T15:13:41,684 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 53401f0681ce510d042c1599ea9bc7e2 2024-12-07T15:13:41,686 INFO [StoreOpener-53401f0681ce510d042c1599ea9bc7e2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 53401f0681ce510d042c1599ea9bc7e2 2024-12-07T15:13:41,690 INFO [StoreOpener-53401f0681ce510d042c1599ea9bc7e2-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-07T15:13:41,690 INFO [StoreOpener-53401f0681ce510d042c1599ea9bc7e2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 53401f0681ce510d042c1599ea9bc7e2 columnFamilyName A 2024-12-07T15:13:41,690 DEBUG [StoreOpener-53401f0681ce510d042c1599ea9bc7e2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:13:41,691 INFO [StoreOpener-53401f0681ce510d042c1599ea9bc7e2-1 {}] regionserver.HStore(327): Store=53401f0681ce510d042c1599ea9bc7e2/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T15:13:41,692 INFO [StoreOpener-53401f0681ce510d042c1599ea9bc7e2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 53401f0681ce510d042c1599ea9bc7e2 2024-12-07T15:13:41,693 INFO [StoreOpener-53401f0681ce510d042c1599ea9bc7e2-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-07T15:13:41,694 INFO [StoreOpener-53401f0681ce510d042c1599ea9bc7e2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 53401f0681ce510d042c1599ea9bc7e2 columnFamilyName B 2024-12-07T15:13:41,694 DEBUG [StoreOpener-53401f0681ce510d042c1599ea9bc7e2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:13:41,695 INFO [StoreOpener-53401f0681ce510d042c1599ea9bc7e2-1 {}] regionserver.HStore(327): Store=53401f0681ce510d042c1599ea9bc7e2/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T15:13:41,695 INFO [StoreOpener-53401f0681ce510d042c1599ea9bc7e2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 53401f0681ce510d042c1599ea9bc7e2 2024-12-07T15:13:41,696 INFO [StoreOpener-53401f0681ce510d042c1599ea9bc7e2-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-07T15:13:41,697 INFO [StoreOpener-53401f0681ce510d042c1599ea9bc7e2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 53401f0681ce510d042c1599ea9bc7e2 columnFamilyName C 2024-12-07T15:13:41,697 DEBUG [StoreOpener-53401f0681ce510d042c1599ea9bc7e2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:13:41,698 INFO [StoreOpener-53401f0681ce510d042c1599ea9bc7e2-1 {}] regionserver.HStore(327): Store=53401f0681ce510d042c1599ea9bc7e2/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T15:13:41,698 INFO [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:41,699 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 53401f0681ce510d042c1599ea9bc7e2 2024-12-07T15:13:41,700 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2 2024-12-07T15:13:41,701 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2 2024-12-07T15:13:41,703 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 53401f0681ce510d042c1599ea9bc7e2 2024-12-07T15:13:41,703 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 53401f0681ce510d042c1599ea9bc7e2 2024-12-07T15:13:41,704 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-07T15:13:41,707 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 53401f0681ce510d042c1599ea9bc7e2 2024-12-07T15:13:41,710 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T15:13:41,711 INFO [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 53401f0681ce510d042c1599ea9bc7e2; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66216553, jitterRate=-0.013296470046043396}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-07T15:13:41,711 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 53401f0681ce510d042c1599ea9bc7e2 2024-12-07T15:13:41,712 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 53401f0681ce510d042c1599ea9bc7e2: Running coprocessor pre-open hook at 1733584421684Writing region info on filesystem at 1733584421684Initializing all the Stores at 1733584421686 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733584421686Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733584421686Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733584421686Cleaning up temporary data from old regions at 1733584421703 (+17 ms)Running coprocessor post-open hooks at 1733584421711 (+8 ms)Region opened successfully at 1733584421712 (+1 ms) 2024-12-07T15:13:41,714 INFO [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., pid=6, masterSystemTime=1733584421676 2024-12-07T15:13:41,717 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:41,718 INFO [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:41,719 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=53401f0681ce510d042c1599ea9bc7e2, regionState=OPEN, openSeqNum=2, regionLocation=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:41,723 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 because future has completed 2024-12-07T15:13:41,729 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-07T15:13:41,729 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 in 204 msec 2024-12-07T15:13:41,733 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-07T15:13:41,733 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=53401f0681ce510d042c1599ea9bc7e2, ASSIGN in 373 msec 2024-12-07T15:13:41,734 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-07T15:13:41,735 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733584421735"}]},"ts":"1733584421735"} 2024-12-07T15:13:41,738 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-07T15:13:41,739 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-07T15:13:41,743 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 484 msec 2024-12-07T15:13:41,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-07T15:13:41,905 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-07T15:13:41,912 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19b2c994, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:13:41,912 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6797ebe9c52,36915,-1 for getting cluster id 2024-12-07T15:13:41,913 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T15:13:41,914 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f8956274-ed33-4e16-82eb-d5d5911ec893' 2024-12-07T15:13:41,915 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T15:13:41,915 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f8956274-ed33-4e16-82eb-d5d5911ec893" 2024-12-07T15:13:41,915 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d417a52, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:13:41,915 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6797ebe9c52,36915,-1] 2024-12-07T15:13:41,915 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T15:13:41,916 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:13:41,917 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49938, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T15:13:41,919 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f9a777e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:13:41,919 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T15:13:41,921 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6797ebe9c52,46009,1733584418401, seqNum=-1] 2024-12-07T15:13:41,921 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T15:13:41,923 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36010, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T15:13:41,925 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-07T15:13:41,926 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is d6797ebe9c52,36915,1733584417704 2024-12-07T15:13:41,927 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@144e408 2024-12-07T15:13:41,929 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-07T15:13:41,931 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49942, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-07T15:13:41,934 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a52ffa9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:13:41,934 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6797ebe9c52,36915,-1 for getting cluster id 2024-12-07T15:13:41,935 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T15:13:41,936 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f8956274-ed33-4e16-82eb-d5d5911ec893' 2024-12-07T15:13:41,936 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T15:13:41,936 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f8956274-ed33-4e16-82eb-d5d5911ec893" 2024-12-07T15:13:41,937 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@175526a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:13:41,937 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6797ebe9c52,36915,-1] 2024-12-07T15:13:41,937 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T15:13:41,937 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:13:41,938 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49956, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T15:13:41,940 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f55d533, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:13:41,940 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51f14cda, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:13:41,940 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6797ebe9c52,36915,-1 for getting cluster id 2024-12-07T15:13:41,941 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T15:13:41,942 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f8956274-ed33-4e16-82eb-d5d5911ec893' 2024-12-07T15:13:41,942 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T15:13:41,942 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f8956274-ed33-4e16-82eb-d5d5911ec893" 2024-12-07T15:13:41,942 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ccb4eda, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:13:41,943 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6797ebe9c52,36915,-1] 2024-12-07T15:13:41,943 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T15:13:41,943 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:13:41,944 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49968, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T15:13:41,945 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fc8436a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:13:41,946 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4201ea29, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:13:41,946 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6797ebe9c52,36915,-1 for getting cluster id 2024-12-07T15:13:41,946 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T15:13:41,947 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f8956274-ed33-4e16-82eb-d5d5911ec893' 2024-12-07T15:13:41,948 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T15:13:41,948 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f8956274-ed33-4e16-82eb-d5d5911ec893" 2024-12-07T15:13:41,948 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10bc6ecb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:13:41,948 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6797ebe9c52,36915,-1] 2024-12-07T15:13:41,948 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T15:13:41,949 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:13:41,951 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49982, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T15:13:41,951 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64257f01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:13:41,952 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2754d9b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:13:41,952 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6797ebe9c52,36915,-1 for getting cluster id 2024-12-07T15:13:41,952 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T15:13:41,954 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f8956274-ed33-4e16-82eb-d5d5911ec893' 2024-12-07T15:13:41,954 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T15:13:41,954 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f8956274-ed33-4e16-82eb-d5d5911ec893" 2024-12-07T15:13:41,954 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a27ead7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:13:41,954 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6797ebe9c52,36915,-1] 2024-12-07T15:13:41,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T15:13:41,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:13:41,959 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49994, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T15:13:41,960 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58577299, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:13:41,960 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57135afb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:13:41,960 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6797ebe9c52,36915,-1 for getting cluster id 2024-12-07T15:13:41,961 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T15:13:41,961 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f8956274-ed33-4e16-82eb-d5d5911ec893' 2024-12-07T15:13:41,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T15:13:41,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f8956274-ed33-4e16-82eb-d5d5911ec893" 2024-12-07T15:13:41,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2aa43e09, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:13:41,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6797ebe9c52,36915,-1] 2024-12-07T15:13:41,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T15:13:41,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:13:41,963 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50018, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T15:13:41,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@792c9a75, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:13:41,966 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a48b566, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:13:41,966 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6797ebe9c52,36915,-1 for getting cluster id 2024-12-07T15:13:41,966 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T15:13:41,967 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f8956274-ed33-4e16-82eb-d5d5911ec893' 2024-12-07T15:13:41,967 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T15:13:41,967 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f8956274-ed33-4e16-82eb-d5d5911ec893" 2024-12-07T15:13:41,968 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@510fbbc9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:13:41,968 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6797ebe9c52,36915,-1] 2024-12-07T15:13:41,968 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T15:13:41,968 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:13:41,969 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50042, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T15:13:41,970 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@663781c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:13:41,970 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14d81b23, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:13:41,970 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6797ebe9c52,36915,-1 for getting cluster id 2024-12-07T15:13:41,970 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T15:13:41,971 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f8956274-ed33-4e16-82eb-d5d5911ec893' 2024-12-07T15:13:41,971 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T15:13:41,972 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f8956274-ed33-4e16-82eb-d5d5911ec893" 2024-12-07T15:13:41,972 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44bee5c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:13:41,972 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6797ebe9c52,36915,-1] 2024-12-07T15:13:41,972 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T15:13:41,972 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:13:41,973 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50068, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T15:13:41,974 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a98f2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:13:41,975 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a046d27, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:13:41,975 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6797ebe9c52,36915,-1 for getting cluster id 2024-12-07T15:13:41,975 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T15:13:41,976 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f8956274-ed33-4e16-82eb-d5d5911ec893' 2024-12-07T15:13:41,976 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T15:13:41,976 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f8956274-ed33-4e16-82eb-d5d5911ec893" 2024-12-07T15:13:41,976 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12f232f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:13:41,976 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6797ebe9c52,36915,-1] 2024-12-07T15:13:41,977 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T15:13:41,977 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:13:41,977 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50078, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T15:13:41,978 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5be3f202, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:13:41,979 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47ef02a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:13:41,979 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6797ebe9c52,36915,-1 for getting cluster id 2024-12-07T15:13:41,979 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T15:13:41,980 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f8956274-ed33-4e16-82eb-d5d5911ec893' 2024-12-07T15:13:41,980 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T15:13:41,980 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f8956274-ed33-4e16-82eb-d5d5911ec893" 2024-12-07T15:13:41,980 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21f437cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:13:41,981 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6797ebe9c52,36915,-1] 2024-12-07T15:13:41,981 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T15:13:41,981 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:13:41,982 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50106, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T15:13:41,982 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a903853, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:13:41,987 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T15:13:41,987 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T15:13:41,988 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T15:13:41,989 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T15:13:41,989 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6797ebe9c52,46009,1733584418401, seqNum=-1] 2024-12-07T15:13:41,989 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6797ebe9c52,46009,1733584418401, seqNum=-1] 2024-12-07T15:13:41,989 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6797ebe9c52,46009,1733584418401, seqNum=-1] 2024-12-07T15:13:41,989 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T15:13:41,989 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T15:13:41,989 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T15:13:41,991 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36018, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T15:13:41,991 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36024, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T15:13:41,991 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36026, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T15:13:41,992 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T15:13:41,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6797ebe9c52,46009,1733584418401, seqNum=-1] 2024-12-07T15:13:41,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T15:13:41,995 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36040, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T15:13:41,998 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T15:13:41,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-12-07T15:13:41,999 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2] 2024-12-07T15:13:41,999 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2] 2024-12-07T15:13:42,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-07T15:13:42,001 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T15:13:42,001 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T15:13:42,002 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6797ebe9c52,46009,1733584418401, seqNum=-1] 2024-12-07T15:13:42,002 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T15:13:42,002 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2] 2024-12-07T15:13:42,003 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T15:13:42,004 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36042, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T15:13:42,004 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2] 2024-12-07T15:13:42,005 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T15:13:42,005 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T15:13:42,007 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6797ebe9c52,46009,1733584418401, seqNum=-1] 2024-12-07T15:13:42,007 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2] 2024-12-07T15:13:42,007 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T15:13:42,009 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36052, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T15:13:42,012 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6797ebe9c52,46009,1733584418401, seqNum=-1] 2024-12-07T15:13:42,012 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T15:13:42,013 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T15:13:42,015 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6797ebe9c52,46009,1733584418401, seqNum=-1] 2024-12-07T15:13:42,015 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T15:13:42,016 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36058, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T15:13:42,017 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T15:13:42,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2] 2024-12-07T15:13:42,019 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36062, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T15:13:42,020 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6797ebe9c52,46009,1733584418401, seqNum=-1] 2024-12-07T15:13:42,021 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T15:13:42,022 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2] 2024-12-07T15:13:42,022 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36068, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T15:13:42,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2] 2024-12-07T15:13:42,025 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2] 2024-12-07T15:13:42,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on 53401f0681ce510d042c1599ea9bc7e2 2024-12-07T15:13:42,084 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 53401f0681ce510d042c1599ea9bc7e2 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-07T15:13:42,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 53401f0681ce510d042c1599ea9bc7e2, store=A 2024-12-07T15:13:42,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:13:42,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 53401f0681ce510d042c1599ea9bc7e2, store=B 2024-12-07T15:13:42,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:13:42,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 53401f0681ce510d042c1599ea9bc7e2, store=C 2024-12-07T15:13:42,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:13:42,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-07T15:13:42,189 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-07T15:13:42,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:42,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. as already flushing 2024-12-07T15:13:42,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:42,203 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:13:42,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:13:42,229 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/A/44e00011a9ee46eaa3b997fac29d50c7 is 50, key is test_row_0/A:col10/1733584422066/Put/seqid=0 2024-12-07T15:13:42,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:13:42,243 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:42,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36040 deadline: 1733584482206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:42,251 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:42,252 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:42,252 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:42,251 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733584422194, max timestamp=1733584422225Num uniques 859; 2024-12-07T15:13:42,253 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-07T15:13:42,254 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733584422194, max timestamp=1733584422225Num uniques 859; 2024-12-07T15:13:42,254 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-07T15:13:42,263 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733584422194, max timestamp=1733584422225Num uniques 859; 2024-12-07T15:13:42,264 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-07T15:13:42,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741837_1013 (size=16681) 2024-12-07T15:13:42,303 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/A/44e00011a9ee46eaa3b997fac29d50c7 2024-12-07T15:13:42,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-07T15:13:42,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:42,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:42,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36068 deadline: 1733584482344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:42,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1733584482348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:42,371 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:42,371 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:42,371 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:42,372 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:42,372 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:42,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:42,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:42,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36026 deadline: 1733584482354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:42,375 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:42,375 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:42,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:42,376 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:42,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36024 deadline: 1733584482360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:42,377 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:42,377 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:42,377 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:42,377 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:42,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36040 deadline: 1733584482364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:42,381 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:42,382 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:42,382 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:42,386 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-07T15:13:42,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:42,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. as already flushing 2024-12-07T15:13:42,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:42,396 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:13:42,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:13:42,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:13:42,453 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/B/05474634a257474ca595ea1ca3c4a52c is 50, key is test_row_0/B:col10/1733584422066/Put/seqid=0 2024-12-07T15:13:42,479 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:42,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1733584482474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:42,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:42,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:42,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741838_1014 (size=12001) 2024-12-07T15:13:42,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:42,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:42,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36068 deadline: 1733584482485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:42,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:42,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36026 deadline: 1733584482485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:42,490 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:42,491 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:42,491 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:42,491 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:42,491 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:42,491 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:42,492 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:42,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36024 deadline: 1733584482486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:42,493 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:42,493 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:42,493 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:42,552 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-07T15:13:42,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:42,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. as already flushing 2024-12-07T15:13:42,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:42,555 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:13:42,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:13:42,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:13:42,599 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:42,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36040 deadline: 1733584482594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:42,600 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:42,600 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:42,601 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:42,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-07T15:13:42,690 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:42,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1733584482684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:42,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:42,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:42,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:42,695 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:42,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36068 deadline: 1733584482694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:42,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:42,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36026 deadline: 1733584482694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:42,702 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:42,702 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:42,702 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:42,702 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:42,702 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:42,703 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:42,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:42,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36024 deadline: 1733584482704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:42,713 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:42,713 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:42,713 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-07T15:13:42,713 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:42,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:42,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. as already flushing 2024-12-07T15:13:42,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:42,714 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:13:42,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:13:42,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:13:42,871 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-07T15:13:42,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:42,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. as already flushing 2024-12-07T15:13:42,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:42,872 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:13:42,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:13:42,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:13:42,883 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/B/05474634a257474ca595ea1ca3c4a52c 2024-12-07T15:13:42,919 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:42,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36040 deadline: 1733584482914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:42,921 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:42,921 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:42,921 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:42,942 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/C/a4f19d5c96b446aea3b5af6db256d1f6 is 50, key is test_row_0/C:col10/1733584422066/Put/seqid=0 2024-12-07T15:13:42,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741839_1015 (size=12001) 2024-12-07T15:13:42,996 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/C/a4f19d5c96b446aea3b5af6db256d1f6 2024-12-07T15:13:43,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:43,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1733584483004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:43,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:43,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:43,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:43,018 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:43,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36026 deadline: 1733584483017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:43,020 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:43,020 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:43,020 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:43,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:43,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36068 deadline: 1733584483019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:43,022 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:43,022 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:43,022 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:43,024 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/A/44e00011a9ee46eaa3b997fac29d50c7 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/44e00011a9ee46eaa3b997fac29d50c7 2024-12-07T15:13:43,027 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-07T15:13:43,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:43,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. as already flushing 2024-12-07T15:13:43,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:43,028 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:13:43,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:13:43,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:13:43,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:43,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36024 deadline: 1733584483024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:43,034 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:43,034 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:43,034 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:43,045 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/44e00011a9ee46eaa3b997fac29d50c7, entries=250, sequenceid=12, filesize=16.3 K 2024-12-07T15:13:43,050 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-12-07T15:13:43,053 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/B/05474634a257474ca595ea1ca3c4a52c as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/05474634a257474ca595ea1ca3c4a52c 2024-12-07T15:13:43,066 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/05474634a257474ca595ea1ca3c4a52c, entries=150, sequenceid=12, filesize=11.7 K 2024-12-07T15:13:43,068 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/C/a4f19d5c96b446aea3b5af6db256d1f6 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/a4f19d5c96b446aea3b5af6db256d1f6 2024-12-07T15:13:43,080 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/a4f19d5c96b446aea3b5af6db256d1f6, entries=150, sequenceid=12, filesize=11.7 K 2024-12-07T15:13:43,094 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 53401f0681ce510d042c1599ea9bc7e2 in 1013ms, sequenceid=12, compaction requested=false 2024-12-07T15:13:43,096 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-07T15:13:43,098 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 53401f0681ce510d042c1599ea9bc7e2: 2024-12-07T15:13:43,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-07T15:13:43,185 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-07T15:13:43,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:43,186 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 53401f0681ce510d042c1599ea9bc7e2 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-12-07T15:13:43,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 53401f0681ce510d042c1599ea9bc7e2, store=A 2024-12-07T15:13:43,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T15:13:43,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 53401f0681ce510d042c1599ea9bc7e2, store=B 2024-12-07T15:13:43,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T15:13:43,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 53401f0681ce510d042c1599ea9bc7e2, store=C 2024-12-07T15:13:43,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T15:13:43,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/A/095c5365ce6d4c43917491448d0870ac is 50, key is test_row_0/A:col10/1733584422310/Put/seqid=0 2024-12-07T15:13:43,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741840_1016 (size=12001) 2024-12-07T15:13:43,241 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/A/095c5365ce6d4c43917491448d0870ac 2024-12-07T15:13:43,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/B/321b66df157743c98b652dc7f6b6c891 is 50, key is test_row_0/B:col10/1733584422310/Put/seqid=0 2024-12-07T15:13:43,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741841_1017 (size=12001) 2024-12-07T15:13:43,305 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/B/321b66df157743c98b652dc7f6b6c891 2024-12-07T15:13:43,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/C/23ead6ba94dc4c7abdb7c471bb08ed4c is 50, key is test_row_0/C:col10/1733584422310/Put/seqid=0 2024-12-07T15:13:43,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741842_1018 (size=12001) 2024-12-07T15:13:43,351 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/C/23ead6ba94dc4c7abdb7c471bb08ed4c 2024-12-07T15:13:43,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/A/095c5365ce6d4c43917491448d0870ac as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/095c5365ce6d4c43917491448d0870ac 2024-12-07T15:13:43,385 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/095c5365ce6d4c43917491448d0870ac, entries=150, sequenceid=48, filesize=11.7 K 2024-12-07T15:13:43,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/B/321b66df157743c98b652dc7f6b6c891 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/321b66df157743c98b652dc7f6b6c891 2024-12-07T15:13:43,426 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/321b66df157743c98b652dc7f6b6c891, entries=150, sequenceid=48, filesize=11.7 K 2024-12-07T15:13:43,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/C/23ead6ba94dc4c7abdb7c471bb08ed4c as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/23ead6ba94dc4c7abdb7c471bb08ed4c 2024-12-07T15:13:43,449 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/23ead6ba94dc4c7abdb7c471bb08ed4c, entries=150, sequenceid=48, filesize=11.7 K 2024-12-07T15:13:43,451 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=0 B/0 for 53401f0681ce510d042c1599ea9bc7e2 in 265ms, sequenceid=48, compaction requested=false 2024-12-07T15:13:43,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 53401f0681ce510d042c1599ea9bc7e2: 2024-12-07T15:13:43,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:43,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-12-07T15:13:43,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-12-07T15:13:43,465 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-07T15:13:43,465 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4500 sec 2024-12-07T15:13:43,472 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 1.4760 sec 2024-12-07T15:13:43,553 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 53401f0681ce510d042c1599ea9bc7e2 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-07T15:13:43,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on 53401f0681ce510d042c1599ea9bc7e2 2024-12-07T15:13:43,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 53401f0681ce510d042c1599ea9bc7e2, store=A 2024-12-07T15:13:43,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:13:43,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 53401f0681ce510d042c1599ea9bc7e2, store=B 2024-12-07T15:13:43,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:13:43,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 53401f0681ce510d042c1599ea9bc7e2, store=C 2024-12-07T15:13:43,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:13:43,583 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/A/2fa6c00101fd417caa3eb8ac1c5b7395 is 50, key is test_row_0/A:col10/1733584423528/Put/seqid=0 2024-12-07T15:13:43,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741843_1019 (size=12001) 2024-12-07T15:13:43,606 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/A/2fa6c00101fd417caa3eb8ac1c5b7395 2024-12-07T15:13:43,639 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/B/d4b8a070cbb848ca88d3f3284c729089 is 50, key is test_row_0/B:col10/1733584423528/Put/seqid=0 2024-12-07T15:13:43,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741844_1020 (size=12001) 2024-12-07T15:13:43,672 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/B/d4b8a070cbb848ca88d3f3284c729089 2024-12-07T15:13:43,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:43,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36040 deadline: 1733584483652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:43,699 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:43,699 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:43,699 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:43,699 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733584423640, max timestamp=1733584423676Num uniques 859; 2024-12-07T15:13:43,700 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-07T15:13:43,705 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733584423640, max timestamp=1733584423684Num uniques 859; 2024-12-07T15:13:43,705 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-07T15:13:43,709 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/C/27890a55df4445a4bcbaba4fe9ab60ba is 50, key is test_row_0/C:col10/1733584423528/Put/seqid=0 2024-12-07T15:13:43,720 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733584423640, max timestamp=1733584423689Num uniques 859; 2024-12-07T15:13:43,720 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-07T15:13:43,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741845_1021 (size=12001) 2024-12-07T15:13:43,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:43,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36068 deadline: 1733584483781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:43,805 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:43,805 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:43,805 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:43,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:43,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1733584483786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:43,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:43,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:43,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:43,818 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:43,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36026 deadline: 1733584483803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:43,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:43,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36040 deadline: 1733584483803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:43,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:43,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36024 deadline: 1733584483803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:43,821 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:43,821 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:43,821 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:43,821 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:43,821 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:43,821 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:43,823 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:43,823 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:43,823 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:43,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:43,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36068 deadline: 1733584483913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:43,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:43,930 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:43,930 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:43,930 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:43,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1733584483914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:43,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:43,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:43,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:43,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:43,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36024 deadline: 1733584483923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:43,934 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:43,934 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:43,934 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:43,934 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:43,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36026 deadline: 1733584483923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:43,935 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:43,935 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:43,935 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:44,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:44,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36040 deadline: 1733584484034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:44,037 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:44,037 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:44,038 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:44,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:44,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36068 deadline: 1733584484133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:44,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:44,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1733584484136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:44,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:44,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:44,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:44,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:44,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:44,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:44,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-07T15:13:44,154 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T15:13:44,155 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T15:13:44,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:44,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36026 deadline: 1733584484143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:44,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-12-07T15:13:44,158 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:44,158 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:44,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:44,158 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:44,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36024 deadline: 1733584484144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:44,159 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:44,159 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:44,159 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:44,161 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T15:13:44,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-07T15:13:44,163 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T15:13:44,163 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T15:13:44,177 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/C/27890a55df4445a4bcbaba4fe9ab60ba 2024-12-07T15:13:44,192 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/A/2fa6c00101fd417caa3eb8ac1c5b7395 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/2fa6c00101fd417caa3eb8ac1c5b7395 2024-12-07T15:13:44,205 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/2fa6c00101fd417caa3eb8ac1c5b7395, entries=150, sequenceid=59, filesize=11.7 K 2024-12-07T15:13:44,208 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/B/d4b8a070cbb848ca88d3f3284c729089 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/d4b8a070cbb848ca88d3f3284c729089 2024-12-07T15:13:44,223 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/d4b8a070cbb848ca88d3f3284c729089, entries=150, sequenceid=59, filesize=11.7 K 2024-12-07T15:13:44,228 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/C/27890a55df4445a4bcbaba4fe9ab60ba as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/27890a55df4445a4bcbaba4fe9ab60ba 2024-12-07T15:13:44,254 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/27890a55df4445a4bcbaba4fe9ab60ba, entries=150, sequenceid=59, filesize=11.7 K 2024-12-07T15:13:44,256 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 53401f0681ce510d042c1599ea9bc7e2 in 703ms, sequenceid=59, compaction requested=true 2024-12-07T15:13:44,256 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 53401f0681ce510d042c1599ea9bc7e2: 2024-12-07T15:13:44,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 53401f0681ce510d042c1599ea9bc7e2:A, priority=-2147483648, current under compaction store size is 1 2024-12-07T15:13:44,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:13:44,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 53401f0681ce510d042c1599ea9bc7e2:B, priority=-2147483648, current under compaction store size is 2 2024-12-07T15:13:44,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:13:44,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 53401f0681ce510d042c1599ea9bc7e2:C, priority=-2147483648, current under compaction store size is 3 2024-12-07T15:13:44,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T15:13:44,276 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:13:44,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-07T15:13:44,280 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:13:44,280 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:13:44,282 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1541): 53401f0681ce510d042c1599ea9bc7e2/B is initiating minor compaction (all files) 2024-12-07T15:13:44,283 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 53401f0681ce510d042c1599ea9bc7e2/B in TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:44,283 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/05474634a257474ca595ea1ca3c4a52c, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/321b66df157743c98b652dc7f6b6c891, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/d4b8a070cbb848ca88d3f3284c729089] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp, totalSize=35.2 K 2024-12-07T15:13:44,284 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:13:44,285 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1541): 53401f0681ce510d042c1599ea9bc7e2/A is initiating minor compaction (all files) 2024-12-07T15:13:44,285 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 53401f0681ce510d042c1599ea9bc7e2/A in TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:44,285 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/44e00011a9ee46eaa3b997fac29d50c7, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/095c5365ce6d4c43917491448d0870ac, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/2fa6c00101fd417caa3eb8ac1c5b7395] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp, totalSize=39.7 K 2024-12-07T15:13:44,286 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 05474634a257474ca595ea1ca3c4a52c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733584422051 2024-12-07T15:13:44,286 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 44e00011a9ee46eaa3b997fac29d50c7, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733584422015 2024-12-07T15:13:44,287 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 321b66df157743c98b652dc7f6b6c891, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1733584422310 2024-12-07T15:13:44,287 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 095c5365ce6d4c43917491448d0870ac, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1733584422310 2024-12-07T15:13:44,288 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2fa6c00101fd417caa3eb8ac1c5b7395, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1733584423528 2024-12-07T15:13:44,288 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting d4b8a070cbb848ca88d3f3284c729089, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1733584423528 2024-12-07T15:13:44,318 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-07T15:13:44,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:44,319 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 53401f0681ce510d042c1599ea9bc7e2 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-12-07T15:13:44,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 53401f0681ce510d042c1599ea9bc7e2, store=A 2024-12-07T15:13:44,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T15:13:44,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 53401f0681ce510d042c1599ea9bc7e2, store=B 2024-12-07T15:13:44,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T15:13:44,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 53401f0681ce510d042c1599ea9bc7e2, store=C 2024-12-07T15:13:44,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T15:13:44,337 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 53401f0681ce510d042c1599ea9bc7e2#B#compaction#9 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:13:44,338 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/B/159141ec8899440bbab8fb159d74aa53 is 50, key is test_row_0/B:col10/1733584423528/Put/seqid=0 2024-12-07T15:13:44,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/A/9fd1b9703f8d435993175c04de4066a3 is 50, key is test_row_0/A:col10/1733584423795/Put/seqid=0 2024-12-07T15:13:44,354 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 53401f0681ce510d042c1599ea9bc7e2#A#compaction#11 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:13:44,355 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/A/beb5d366a81b4fbc81569bbea20bdfb5 is 50, key is test_row_0/A:col10/1733584423528/Put/seqid=0 2024-12-07T15:13:44,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on 53401f0681ce510d042c1599ea9bc7e2 2024-12-07T15:13:44,358 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. as already flushing 2024-12-07T15:13:44,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741846_1022 (size=12104) 2024-12-07T15:13:44,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741848_1024 (size=12104) 2024-12-07T15:13:44,409 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/A/beb5d366a81b4fbc81569bbea20bdfb5 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/beb5d366a81b4fbc81569bbea20bdfb5 2024-12-07T15:13:44,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741847_1023 (size=12001) 2024-12-07T15:13:44,419 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/A/9fd1b9703f8d435993175c04de4066a3 2024-12-07T15:13:44,442 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 53401f0681ce510d042c1599ea9bc7e2/A of 53401f0681ce510d042c1599ea9bc7e2 into beb5d366a81b4fbc81569bbea20bdfb5(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:13:44,442 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 53401f0681ce510d042c1599ea9bc7e2: 2024-12-07T15:13:44,446 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., storeName=53401f0681ce510d042c1599ea9bc7e2/A, priority=13, startTime=1733584424258; duration=0sec 2024-12-07T15:13:44,446 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T15:13:44,446 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 53401f0681ce510d042c1599ea9bc7e2:A 2024-12-07T15:13:44,447 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:13:44,449 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:13:44,449 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1541): 53401f0681ce510d042c1599ea9bc7e2/C is initiating minor compaction (all files) 2024-12-07T15:13:44,450 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 53401f0681ce510d042c1599ea9bc7e2/C in TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:44,450 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/a4f19d5c96b446aea3b5af6db256d1f6, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/23ead6ba94dc4c7abdb7c471bb08ed4c, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/27890a55df4445a4bcbaba4fe9ab60ba] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp, totalSize=35.2 K 2024-12-07T15:13:44,451 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting a4f19d5c96b446aea3b5af6db256d1f6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733584422051 2024-12-07T15:13:44,452 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23ead6ba94dc4c7abdb7c471bb08ed4c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1733584422310 2024-12-07T15:13:44,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/B/7d5862eb5798423e9fe79bde85746d66 is 50, key is test_row_0/B:col10/1733584423795/Put/seqid=0 2024-12-07T15:13:44,455 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 27890a55df4445a4bcbaba4fe9ab60ba, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1733584423528 2024-12-07T15:13:44,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:44,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36068 deadline: 1733584484443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:44,457 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:44,457 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:44,457 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:44,461 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:44,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36040 deadline: 1733584484444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:44,462 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:44,463 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:44,463 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:44,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:44,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1733584484457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:44,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:44,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:44,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:44,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:44,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36026 deadline: 1733584484464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:44,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:44,470 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:44,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36024 deadline: 1733584484464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:44,471 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:44,471 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:44,471 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:44,471 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:44,471 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:44,481 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 53401f0681ce510d042c1599ea9bc7e2#C#compaction#13 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:13:44,483 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/C/3e33d751ad924ded995c3dad74cf5e05 is 50, key is test_row_0/C:col10/1733584423528/Put/seqid=0 2024-12-07T15:13:44,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-07T15:13:44,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741849_1025 (size=12001) 2024-12-07T15:13:44,509 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/B/7d5862eb5798423e9fe79bde85746d66 2024-12-07T15:13:44,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/C/0d0b65f94e1f4a9dbaacd46759426d55 is 50, key is test_row_0/C:col10/1733584423795/Put/seqid=0 2024-12-07T15:13:44,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741850_1026 (size=12104) 2024-12-07T15:13:44,574 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/C/3e33d751ad924ded995c3dad74cf5e05 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/3e33d751ad924ded995c3dad74cf5e05 2024-12-07T15:13:44,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741851_1027 (size=12001) 2024-12-07T15:13:44,581 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/C/0d0b65f94e1f4a9dbaacd46759426d55 2024-12-07T15:13:44,582 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:44,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36040 deadline: 1733584484573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:44,583 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:44,583 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:44,583 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:44,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:44,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1733584484574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:44,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:44,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:44,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:44,592 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 53401f0681ce510d042c1599ea9bc7e2/C of 53401f0681ce510d042c1599ea9bc7e2 into 3e33d751ad924ded995c3dad74cf5e05(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:13:44,592 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 53401f0681ce510d042c1599ea9bc7e2: 2024-12-07T15:13:44,593 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., storeName=53401f0681ce510d042c1599ea9bc7e2/C, priority=13, startTime=1733584424271; duration=0sec 2024-12-07T15:13:44,593 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:13:44,593 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 53401f0681ce510d042c1599ea9bc7e2:C 2024-12-07T15:13:44,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/A/9fd1b9703f8d435993175c04de4066a3 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/9fd1b9703f8d435993175c04de4066a3 2024-12-07T15:13:44,606 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/9fd1b9703f8d435993175c04de4066a3, entries=150, sequenceid=95, filesize=11.7 K 2024-12-07T15:13:44,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/B/7d5862eb5798423e9fe79bde85746d66 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/7d5862eb5798423e9fe79bde85746d66 2024-12-07T15:13:44,620 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/7d5862eb5798423e9fe79bde85746d66, entries=150, sequenceid=95, filesize=11.7 K 2024-12-07T15:13:44,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/C/0d0b65f94e1f4a9dbaacd46759426d55 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/0d0b65f94e1f4a9dbaacd46759426d55 2024-12-07T15:13:44,635 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/0d0b65f94e1f4a9dbaacd46759426d55, entries=150, sequenceid=95, filesize=11.7 K 2024-12-07T15:13:44,637 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for 53401f0681ce510d042c1599ea9bc7e2 in 318ms, sequenceid=95, compaction requested=false 2024-12-07T15:13:44,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 53401f0681ce510d042c1599ea9bc7e2: 2024-12-07T15:13:44,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:44,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-12-07T15:13:44,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-12-07T15:13:44,645 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-12-07T15:13:44,645 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 477 msec 2024-12-07T15:13:44,649 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 491 msec 2024-12-07T15:13:44,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-07T15:13:44,795 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T15:13:44,796 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T15:13:44,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-12-07T15:13:44,800 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 53401f0681ce510d042c1599ea9bc7e2 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-07T15:13:44,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 53401f0681ce510d042c1599ea9bc7e2, store=A 2024-12-07T15:13:44,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:13:44,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 53401f0681ce510d042c1599ea9bc7e2, store=B 2024-12-07T15:13:44,801 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T15:13:44,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:13:44,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 53401f0681ce510d042c1599ea9bc7e2, store=C 2024-12-07T15:13:44,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:13:44,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on 53401f0681ce510d042c1599ea9bc7e2 2024-12-07T15:13:44,803 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T15:13:44,803 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T15:13:44,804 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/B/159141ec8899440bbab8fb159d74aa53 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/159141ec8899440bbab8fb159d74aa53 2024-12-07T15:13:44,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-07T15:13:44,819 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/A/2cb709721cbd4613bf10a9064351a369 is 50, key is test_row_0/A:col10/1733584424413/Put/seqid=0 2024-12-07T15:13:44,821 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 53401f0681ce510d042c1599ea9bc7e2/B of 53401f0681ce510d042c1599ea9bc7e2 into 159141ec8899440bbab8fb159d74aa53(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:13:44,821 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 53401f0681ce510d042c1599ea9bc7e2: 2024-12-07T15:13:44,821 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., storeName=53401f0681ce510d042c1599ea9bc7e2/B, priority=13, startTime=1733584424271; duration=0sec 2024-12-07T15:13:44,822 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:13:44,822 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 53401f0681ce510d042c1599ea9bc7e2:B 2024-12-07T15:13:44,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741852_1028 (size=14341) 2024-12-07T15:13:44,841 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/A/2cb709721cbd4613bf10a9064351a369 2024-12-07T15:13:44,876 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/B/e6221ebb109f42729c4b05800d1f65a2 is 50, key is test_row_0/B:col10/1733584424413/Put/seqid=0 2024-12-07T15:13:44,906 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:44,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1733584484901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:44,908 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:44,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36040 deadline: 1733584484904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:44,909 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:44,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:44,909 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:44,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:44,909 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:44,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:44,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-07T15:13:44,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741853_1029 (size=12001) 2024-12-07T15:13:44,933 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/B/e6221ebb109f42729c4b05800d1f65a2 2024-12-07T15:13:44,952 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/C/b2c647e6d0d249dcb1f59ddc8c022ced is 50, key is test_row_0/C:col10/1733584424413/Put/seqid=0 2024-12-07T15:13:44,963 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-07T15:13:44,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:44,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. as already flushing 2024-12-07T15:13:44,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:44,964 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:13:44,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:13:44,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:13:44,968 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:44,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36068 deadline: 1733584484963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:44,969 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:44,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:44,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:44,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741854_1030 (size=12001) 2024-12-07T15:13:44,980 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/C/b2c647e6d0d249dcb1f59ddc8c022ced 2024-12-07T15:13:44,986 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:44,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36026 deadline: 1733584484984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:44,987 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:44,988 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:44,988 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:44,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:44,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36024 deadline: 1733584484985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:44,989 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:44,989 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:44,989 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:44,996 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/A/2cb709721cbd4613bf10a9064351a369 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/2cb709721cbd4613bf10a9064351a369 2024-12-07T15:13:45,007 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/2cb709721cbd4613bf10a9064351a369, entries=200, sequenceid=109, filesize=14.0 K 2024-12-07T15:13:45,009 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/B/e6221ebb109f42729c4b05800d1f65a2 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/e6221ebb109f42729c4b05800d1f65a2 2024-12-07T15:13:45,014 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:45,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36040 deadline: 1733584485013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:45,015 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:45,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:45,015 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:45,015 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:45,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1733584485013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:45,016 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:45,016 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:45,016 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:45,027 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/e6221ebb109f42729c4b05800d1f65a2, entries=150, sequenceid=109, filesize=11.7 K 2024-12-07T15:13:45,029 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/C/b2c647e6d0d249dcb1f59ddc8c022ced as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/b2c647e6d0d249dcb1f59ddc8c022ced 2024-12-07T15:13:45,040 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/b2c647e6d0d249dcb1f59ddc8c022ced, entries=150, sequenceid=109, filesize=11.7 K 2024-12-07T15:13:45,042 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 53401f0681ce510d042c1599ea9bc7e2 in 242ms, sequenceid=109, compaction requested=true 2024-12-07T15:13:45,042 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 53401f0681ce510d042c1599ea9bc7e2: 2024-12-07T15:13:45,042 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 53401f0681ce510d042c1599ea9bc7e2:A, priority=-2147483648, current under compaction store size is 1 2024-12-07T15:13:45,042 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:13:45,042 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:13:45,043 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:13:45,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 53401f0681ce510d042c1599ea9bc7e2:B, priority=-2147483648, current under compaction store size is 2 2024-12-07T15:13:45,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:13:45,045 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:13:45,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 53401f0681ce510d042c1599ea9bc7e2:C, priority=-2147483648, current under compaction store size is 3 2024-12-07T15:13:45,045 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1541): 53401f0681ce510d042c1599ea9bc7e2/A is initiating minor compaction (all files) 2024-12-07T15:13:45,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T15:13:45,045 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 53401f0681ce510d042c1599ea9bc7e2/A in TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:45,045 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/beb5d366a81b4fbc81569bbea20bdfb5, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/9fd1b9703f8d435993175c04de4066a3, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/2cb709721cbd4613bf10a9064351a369] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp, totalSize=37.5 K 2024-12-07T15:13:45,046 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:13:45,046 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1541): 53401f0681ce510d042c1599ea9bc7e2/B is initiating minor compaction (all files) 2024-12-07T15:13:45,046 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 53401f0681ce510d042c1599ea9bc7e2/B in TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:45,046 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/159141ec8899440bbab8fb159d74aa53, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/7d5862eb5798423e9fe79bde85746d66, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/e6221ebb109f42729c4b05800d1f65a2] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp, totalSize=35.3 K 2024-12-07T15:13:45,047 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting beb5d366a81b4fbc81569bbea20bdfb5, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1733584423528 2024-12-07T15:13:45,048 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9fd1b9703f8d435993175c04de4066a3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733584423753 2024-12-07T15:13:45,049 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2cb709721cbd4613bf10a9064351a369, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1733584424413 2024-12-07T15:13:45,049 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 159141ec8899440bbab8fb159d74aa53, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1733584423528 2024-12-07T15:13:45,051 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d5862eb5798423e9fe79bde85746d66, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733584423753 2024-12-07T15:13:45,054 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting e6221ebb109f42729c4b05800d1f65a2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1733584424413 2024-12-07T15:13:45,075 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 53401f0681ce510d042c1599ea9bc7e2#A#compaction#18 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:13:45,077 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/A/a072b7d4a5c249a7828f3bec3ca4bf21 is 50, key is test_row_0/A:col10/1733584424413/Put/seqid=0 2024-12-07T15:13:45,084 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 53401f0681ce510d042c1599ea9bc7e2#B#compaction#19 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:13:45,085 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/B/826837eaedcb40dda69be772defc0c0a is 50, key is test_row_0/B:col10/1733584424413/Put/seqid=0 2024-12-07T15:13:45,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741855_1031 (size=12207) 2024-12-07T15:13:45,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741856_1032 (size=12207) 2024-12-07T15:13:45,120 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-07T15:13:45,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:45,121 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 53401f0681ce510d042c1599ea9bc7e2 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-07T15:13:45,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 53401f0681ce510d042c1599ea9bc7e2, store=A 2024-12-07T15:13:45,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:13:45,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 53401f0681ce510d042c1599ea9bc7e2, store=B 2024-12-07T15:13:45,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:13:45,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 53401f0681ce510d042c1599ea9bc7e2, store=C 2024-12-07T15:13:45,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:13:45,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-07T15:13:45,129 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/B/826837eaedcb40dda69be772defc0c0a as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/826837eaedcb40dda69be772defc0c0a 2024-12-07T15:13:45,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/A/1e21c007e21147dbaf96246b1c5d2b78 is 50, key is test_row_0/A:col10/1733584424898/Put/seqid=0 2024-12-07T15:13:45,142 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 53401f0681ce510d042c1599ea9bc7e2/B of 53401f0681ce510d042c1599ea9bc7e2 into 826837eaedcb40dda69be772defc0c0a(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:13:45,143 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 53401f0681ce510d042c1599ea9bc7e2: 2024-12-07T15:13:45,143 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., storeName=53401f0681ce510d042c1599ea9bc7e2/B, priority=13, startTime=1733584425043; duration=0sec 2024-12-07T15:13:45,143 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T15:13:45,143 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 53401f0681ce510d042c1599ea9bc7e2:B 2024-12-07T15:13:45,143 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:13:45,147 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:13:45,147 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1541): 53401f0681ce510d042c1599ea9bc7e2/C is initiating minor compaction (all files) 2024-12-07T15:13:45,147 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 53401f0681ce510d042c1599ea9bc7e2/C in TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:45,147 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/3e33d751ad924ded995c3dad74cf5e05, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/0d0b65f94e1f4a9dbaacd46759426d55, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/b2c647e6d0d249dcb1f59ddc8c022ced] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp, totalSize=35.3 K 2024-12-07T15:13:45,148 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e33d751ad924ded995c3dad74cf5e05, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1733584423528 2024-12-07T15:13:45,149 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d0b65f94e1f4a9dbaacd46759426d55, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733584423753 2024-12-07T15:13:45,149 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting b2c647e6d0d249dcb1f59ddc8c022ced, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1733584424413 2024-12-07T15:13:45,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741857_1033 (size=12051) 2024-12-07T15:13:45,178 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/A/1e21c007e21147dbaf96246b1c5d2b78 2024-12-07T15:13:45,179 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 53401f0681ce510d042c1599ea9bc7e2#C#compaction#21 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:13:45,181 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/C/4fa4df3f7c02452fbdcee3e338c8f2dd is 50, key is test_row_0/C:col10/1733584424413/Put/seqid=0 2024-12-07T15:13:45,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741858_1034 (size=12207) 2024-12-07T15:13:45,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/B/0dd77d70596245cf91b2cd01af0e9596 is 50, key is test_row_0/B:col10/1733584424898/Put/seqid=0 2024-12-07T15:13:45,214 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/C/4fa4df3f7c02452fbdcee3e338c8f2dd as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/4fa4df3f7c02452fbdcee3e338c8f2dd 2024-12-07T15:13:45,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on 53401f0681ce510d042c1599ea9bc7e2 2024-12-07T15:13:45,227 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. as already flushing 2024-12-07T15:13:45,227 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 53401f0681ce510d042c1599ea9bc7e2/C of 53401f0681ce510d042c1599ea9bc7e2 into 4fa4df3f7c02452fbdcee3e338c8f2dd(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:13:45,227 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 53401f0681ce510d042c1599ea9bc7e2: 2024-12-07T15:13:45,227 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., storeName=53401f0681ce510d042c1599ea9bc7e2/C, priority=13, startTime=1733584425045; duration=0sec 2024-12-07T15:13:45,228 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:13:45,228 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 53401f0681ce510d042c1599ea9bc7e2:C 2024-12-07T15:13:45,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741859_1035 (size=12051) 2024-12-07T15:13:45,243 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/B/0dd77d70596245cf91b2cd01af0e9596 2024-12-07T15:13:45,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:45,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1733584485261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:45,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:45,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:45,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:45,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36040 deadline: 1733584485262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:45,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:45,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/C/d8fe0170e6d8410084f1fdf573091f8a is 50, key is test_row_0/C:col10/1733584424898/Put/seqid=0 2024-12-07T15:13:45,268 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:45,268 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:45,268 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:45,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741860_1036 (size=12051) 2024-12-07T15:13:45,310 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/C/d8fe0170e6d8410084f1fdf573091f8a 2024-12-07T15:13:45,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/A/1e21c007e21147dbaf96246b1c5d2b78 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/1e21c007e21147dbaf96246b1c5d2b78 2024-12-07T15:13:45,332 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/1e21c007e21147dbaf96246b1c5d2b78, entries=150, sequenceid=130, filesize=11.8 K 2024-12-07T15:13:45,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/B/0dd77d70596245cf91b2cd01af0e9596 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/0dd77d70596245cf91b2cd01af0e9596 2024-12-07T15:13:45,348 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/0dd77d70596245cf91b2cd01af0e9596, entries=150, sequenceid=130, filesize=11.8 K 2024-12-07T15:13:45,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/C/d8fe0170e6d8410084f1fdf573091f8a as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/d8fe0170e6d8410084f1fdf573091f8a 2024-12-07T15:13:45,377 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/d8fe0170e6d8410084f1fdf573091f8a, entries=150, sequenceid=130, filesize=11.8 K 2024-12-07T15:13:45,379 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 53401f0681ce510d042c1599ea9bc7e2 in 259ms, sequenceid=130, compaction requested=false 2024-12-07T15:13:45,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 53401f0681ce510d042c1599ea9bc7e2: 2024-12-07T15:13:45,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:45,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-12-07T15:13:45,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-12-07T15:13:45,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on 53401f0681ce510d042c1599ea9bc7e2 2024-12-07T15:13:45,382 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 53401f0681ce510d042c1599ea9bc7e2 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-07T15:13:45,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 53401f0681ce510d042c1599ea9bc7e2, store=A 2024-12-07T15:13:45,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:13:45,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 53401f0681ce510d042c1599ea9bc7e2, store=B 2024-12-07T15:13:45,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:13:45,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 53401f0681ce510d042c1599ea9bc7e2, store=C 2024-12-07T15:13:45,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:13:45,389 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-12-07T15:13:45,389 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 579 msec 2024-12-07T15:13:45,395 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/A/e67d1910a4a84db7b30984719ae004b4 is 50, key is test_row_0/A:col10/1733584425379/Put/seqid=0 2024-12-07T15:13:45,404 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 594 msec 2024-12-07T15:13:45,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-07T15:13:45,434 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T15:13:45,435 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T15:13:45,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-12-07T15:13:45,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-07T15:13:45,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741861_1037 (size=12151) 2024-12-07T15:13:45,439 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T15:13:45,441 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T15:13:45,441 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/A/e67d1910a4a84db7b30984719ae004b4 2024-12-07T15:13:45,441 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T15:13:45,471 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/B/c164145fd7cc40358f7627dcf9d655e2 is 50, key is test_row_0/B:col10/1733584425379/Put/seqid=0 2024-12-07T15:13:45,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741862_1038 (size=12151) 2024-12-07T15:13:45,512 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/A/a072b7d4a5c249a7828f3bec3ca4bf21 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/a072b7d4a5c249a7828f3bec3ca4bf21 2024-12-07T15:13:45,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:45,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1733584485514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:45,525 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:45,525 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:45,525 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:45,525 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:45,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36040 deadline: 1733584485518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:45,526 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:45,526 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:45,526 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:45,529 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 53401f0681ce510d042c1599ea9bc7e2/A of 53401f0681ce510d042c1599ea9bc7e2 into a072b7d4a5c249a7828f3bec3ca4bf21(size=11.9 K), total size for store is 23.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:13:45,529 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 53401f0681ce510d042c1599ea9bc7e2: 2024-12-07T15:13:45,529 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., storeName=53401f0681ce510d042c1599ea9bc7e2/A, priority=13, startTime=1733584425042; duration=0sec 2024-12-07T15:13:45,529 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:13:45,529 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 53401f0681ce510d042c1599ea9bc7e2:A 2024-12-07T15:13:45,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-07T15:13:45,597 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-07T15:13:45,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:45,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. as already flushing 2024-12-07T15:13:45,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:45,598 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:13:45,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:13:45,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:13:45,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:45,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1733584485634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:45,638 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:45,638 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:45,638 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:45,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:45,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36040 deadline: 1733584485634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:45,640 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:45,640 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:45,640 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:45,753 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-07T15:13:45,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-07T15:13:45,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:45,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. as already flushing 2024-12-07T15:13:45,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:45,754 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:13:45,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:13:45,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:13:45,845 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:45,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1733584485844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:45,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:45,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36040 deadline: 1733584485845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:45,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:45,847 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:45,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:45,847 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:45,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:45,847 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:45,909 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-07T15:13:45,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:45,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. as already flushing 2024-12-07T15:13:45,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:45,910 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:13:45,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:13:45,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:13:45,913 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/B/c164145fd7cc40358f7627dcf9d655e2 2024-12-07T15:13:45,931 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/C/6dc0ef423acb42d7995e4ab54623c53e is 50, key is test_row_0/C:col10/1733584425379/Put/seqid=0 2024-12-07T15:13:45,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741863_1039 (size=12151) 2024-12-07T15:13:45,949 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/C/6dc0ef423acb42d7995e4ab54623c53e 2024-12-07T15:13:45,961 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/A/e67d1910a4a84db7b30984719ae004b4 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/e67d1910a4a84db7b30984719ae004b4 2024-12-07T15:13:45,979 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/e67d1910a4a84db7b30984719ae004b4, entries=150, sequenceid=145, filesize=11.9 K 2024-12-07T15:13:45,981 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/B/c164145fd7cc40358f7627dcf9d655e2 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/c164145fd7cc40358f7627dcf9d655e2 2024-12-07T15:13:45,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:45,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36068 deadline: 1733584485983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:45,992 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to d6797ebe9c52,46009,1733584418401 for 'test_row_0' in 53401f0681ce510d042c1599ea9bc7e2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2207 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-07T15:13:45,993 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:45,993 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:45,993 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:45,996 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/c164145fd7cc40358f7627dcf9d655e2, entries=150, sequenceid=145, filesize=11.9 K 2024-12-07T15:13:45,997 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/C/6dc0ef423acb42d7995e4ab54623c53e as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/6dc0ef423acb42d7995e4ab54623c53e 2024-12-07T15:13:46,006 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:46,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36026 deadline: 1733584486004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:46,007 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:46,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36024 deadline: 1733584486004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:46,007 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/6dc0ef423acb42d7995e4ab54623c53e, entries=150, sequenceid=145, filesize=11.9 K 2024-12-07T15:13:46,007 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to d6797ebe9c52,46009,1733584418401 for 'test_row_1' in 53401f0681ce510d042c1599ea9bc7e2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2204 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-07T15:13:46,008 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:46,008 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:46,008 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:46,008 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to d6797ebe9c52,46009,1733584418401 for 'test_row_0' in 53401f0681ce510d042c1599ea9bc7e2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2206 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-07T15:13:46,009 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:46,009 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:46,009 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:46,010 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 53401f0681ce510d042c1599ea9bc7e2 in 628ms, sequenceid=145, compaction requested=true 2024-12-07T15:13:46,010 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 53401f0681ce510d042c1599ea9bc7e2: 2024-12-07T15:13:46,011 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:13:46,015 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36409 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:13:46,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 53401f0681ce510d042c1599ea9bc7e2:A, priority=-2147483648, current under compaction store size is 1 2024-12-07T15:13:46,015 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1541): 53401f0681ce510d042c1599ea9bc7e2/A is initiating minor compaction (all files) 2024-12-07T15:13:46,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:13:46,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 53401f0681ce510d042c1599ea9bc7e2:B, priority=-2147483648, current under compaction store size is 2 2024-12-07T15:13:46,015 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 53401f0681ce510d042c1599ea9bc7e2/A in TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:46,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T15:13:46,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 53401f0681ce510d042c1599ea9bc7e2:C, priority=-2147483648, current under compaction store size is 3 2024-12-07T15:13:46,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-07T15:13:46,015 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/a072b7d4a5c249a7828f3bec3ca4bf21, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/1e21c007e21147dbaf96246b1c5d2b78, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/e67d1910a4a84db7b30984719ae004b4] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp, totalSize=35.6 K 2024-12-07T15:13:46,016 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:13:46,016 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting a072b7d4a5c249a7828f3bec3ca4bf21, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1733584424413 2024-12-07T15:13:46,017 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e21c007e21147dbaf96246b1c5d2b78, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1733584424889 2024-12-07T15:13:46,018 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting e67d1910a4a84db7b30984719ae004b4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1733584425251 2024-12-07T15:13:46,018 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36409 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:13:46,018 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1541): 53401f0681ce510d042c1599ea9bc7e2/B is initiating minor compaction (all files) 2024-12-07T15:13:46,018 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 53401f0681ce510d042c1599ea9bc7e2/B in TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:46,019 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/826837eaedcb40dda69be772defc0c0a, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/0dd77d70596245cf91b2cd01af0e9596, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/c164145fd7cc40358f7627dcf9d655e2] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp, totalSize=35.6 K 2024-12-07T15:13:46,020 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 826837eaedcb40dda69be772defc0c0a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1733584424413 2024-12-07T15:13:46,022 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 0dd77d70596245cf91b2cd01af0e9596, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1733584424889 2024-12-07T15:13:46,022 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting c164145fd7cc40358f7627dcf9d655e2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1733584425251 2024-12-07T15:13:46,043 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 53401f0681ce510d042c1599ea9bc7e2#A#compaction#27 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:13:46,043 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/A/89cd7191e60346948ebc9cd6b71d587a is 50, key is test_row_0/A:col10/1733584425379/Put/seqid=0 2024-12-07T15:13:46,052 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 53401f0681ce510d042c1599ea9bc7e2#B#compaction#28 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:13:46,052 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/B/42d85a580a414589a2eb4a55b248f749 is 50, key is test_row_0/B:col10/1733584425379/Put/seqid=0 2024-12-07T15:13:46,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-07T15:13:46,066 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-07T15:13:46,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:46,067 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 53401f0681ce510d042c1599ea9bc7e2 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-07T15:13:46,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 53401f0681ce510d042c1599ea9bc7e2, store=A 2024-12-07T15:13:46,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:13:46,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 53401f0681ce510d042c1599ea9bc7e2, store=B 2024-12-07T15:13:46,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:13:46,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 53401f0681ce510d042c1599ea9bc7e2, store=C 2024-12-07T15:13:46,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:13:46,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741864_1040 (size=12459) 2024-12-07T15:13:46,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/A/bab8cfd5cb0949ac887ba3dafceb217e is 50, key is test_row_0/A:col10/1733584425513/Put/seqid=0 2024-12-07T15:13:46,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741865_1041 (size=12459) 2024-12-07T15:13:46,114 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/A/89cd7191e60346948ebc9cd6b71d587a as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/89cd7191e60346948ebc9cd6b71d587a 2024-12-07T15:13:46,118 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/B/42d85a580a414589a2eb4a55b248f749 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/42d85a580a414589a2eb4a55b248f749 2024-12-07T15:13:46,131 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 53401f0681ce510d042c1599ea9bc7e2/A of 53401f0681ce510d042c1599ea9bc7e2 into 89cd7191e60346948ebc9cd6b71d587a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:13:46,132 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 53401f0681ce510d042c1599ea9bc7e2: 2024-12-07T15:13:46,132 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., storeName=53401f0681ce510d042c1599ea9bc7e2/A, priority=13, startTime=1733584426011; duration=0sec 2024-12-07T15:13:46,132 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T15:13:46,132 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 53401f0681ce510d042c1599ea9bc7e2:A 2024-12-07T15:13:46,133 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:13:46,134 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 53401f0681ce510d042c1599ea9bc7e2/B of 53401f0681ce510d042c1599ea9bc7e2 into 42d85a580a414589a2eb4a55b248f749(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:13:46,134 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 53401f0681ce510d042c1599ea9bc7e2: 2024-12-07T15:13:46,134 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., storeName=53401f0681ce510d042c1599ea9bc7e2/B, priority=13, startTime=1733584426015; duration=0sec 2024-12-07T15:13:46,134 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:13:46,134 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 53401f0681ce510d042c1599ea9bc7e2:B 2024-12-07T15:13:46,135 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36409 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:13:46,135 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1541): 53401f0681ce510d042c1599ea9bc7e2/C is initiating minor compaction (all files) 2024-12-07T15:13:46,135 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 53401f0681ce510d042c1599ea9bc7e2/C in TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:46,135 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/4fa4df3f7c02452fbdcee3e338c8f2dd, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/d8fe0170e6d8410084f1fdf573091f8a, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/6dc0ef423acb42d7995e4ab54623c53e] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp, totalSize=35.6 K 2024-12-07T15:13:46,136 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4fa4df3f7c02452fbdcee3e338c8f2dd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1733584424413 2024-12-07T15:13:46,137 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8fe0170e6d8410084f1fdf573091f8a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1733584424889 2024-12-07T15:13:46,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741866_1042 (size=12151) 2024-12-07T15:13:46,138 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6dc0ef423acb42d7995e4ab54623c53e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1733584425251 2024-12-07T15:13:46,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on 53401f0681ce510d042c1599ea9bc7e2 2024-12-07T15:13:46,160 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. as already flushing 2024-12-07T15:13:46,172 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 53401f0681ce510d042c1599ea9bc7e2#C#compaction#30 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:13:46,173 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/C/55b743c9a9064bd3950d11953a239329 is 50, key is test_row_0/C:col10/1733584425379/Put/seqid=0 2024-12-07T15:13:46,195 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:46,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36040 deadline: 1733584486191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:46,196 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:46,196 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:46,197 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:46,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:46,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1733584486194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:46,200 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:46,200 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:46,200 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:46,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741867_1043 (size=12459) 2024-12-07T15:13:46,223 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/C/55b743c9a9064bd3950d11953a239329 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/55b743c9a9064bd3950d11953a239329 2024-12-07T15:13:46,226 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-07T15:13:46,242 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 53401f0681ce510d042c1599ea9bc7e2/C of 53401f0681ce510d042c1599ea9bc7e2 into 55b743c9a9064bd3950d11953a239329(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:13:46,242 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 53401f0681ce510d042c1599ea9bc7e2: 2024-12-07T15:13:46,242 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., storeName=53401f0681ce510d042c1599ea9bc7e2/C, priority=13, startTime=1733584426015; duration=0sec 2024-12-07T15:13:46,243 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:13:46,243 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 53401f0681ce510d042c1599ea9bc7e2:C 2024-12-07T15:13:46,307 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-07T15:13:46,311 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:46,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36040 deadline: 1733584486304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:46,312 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:46,312 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:46,312 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:46,313 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:46,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1733584486304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:46,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:46,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:46,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:46,528 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:46,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:46,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1733584486525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:46,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36040 deadline: 1733584486525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:46,529 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:46,529 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:46,530 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:46,530 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:46,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:46,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:46,540 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/A/bab8cfd5cb0949ac887ba3dafceb217e 2024-12-07T15:13:46,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/B/fc74713e99af49be9a1c3fb3f4b15798 is 50, key is test_row_0/B:col10/1733584425513/Put/seqid=0 2024-12-07T15:13:46,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-07T15:13:46,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741868_1044 (size=12151) 2024-12-07T15:13:46,575 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/B/fc74713e99af49be9a1c3fb3f4b15798 2024-12-07T15:13:46,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/C/b76cd182600f47e29d58b3b617198a8b is 50, key is test_row_0/C:col10/1733584425513/Put/seqid=0 2024-12-07T15:13:46,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741869_1045 (size=12151) 2024-12-07T15:13:46,642 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/C/b76cd182600f47e29d58b3b617198a8b 2024-12-07T15:13:46,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/A/bab8cfd5cb0949ac887ba3dafceb217e as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/bab8cfd5cb0949ac887ba3dafceb217e 2024-12-07T15:13:46,691 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/bab8cfd5cb0949ac887ba3dafceb217e, entries=150, sequenceid=165, filesize=11.9 K 2024-12-07T15:13:46,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/B/fc74713e99af49be9a1c3fb3f4b15798 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/fc74713e99af49be9a1c3fb3f4b15798 2024-12-07T15:13:46,707 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/fc74713e99af49be9a1c3fb3f4b15798, entries=150, sequenceid=165, filesize=11.9 K 2024-12-07T15:13:46,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/C/b76cd182600f47e29d58b3b617198a8b as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/b76cd182600f47e29d58b3b617198a8b 2024-12-07T15:13:46,730 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/b76cd182600f47e29d58b3b617198a8b, entries=150, sequenceid=165, filesize=11.9 K 2024-12-07T15:13:46,732 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 53401f0681ce510d042c1599ea9bc7e2 in 666ms, sequenceid=165, compaction requested=false 2024-12-07T15:13:46,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 53401f0681ce510d042c1599ea9bc7e2: 2024-12-07T15:13:46,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:46,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-12-07T15:13:46,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-12-07T15:13:46,741 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-07T15:13:46,741 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2960 sec 2024-12-07T15:13:46,745 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.3070 sec 2024-12-07T15:13:46,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on 53401f0681ce510d042c1599ea9bc7e2 2024-12-07T15:13:46,837 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 53401f0681ce510d042c1599ea9bc7e2 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-07T15:13:46,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 53401f0681ce510d042c1599ea9bc7e2, store=A 2024-12-07T15:13:46,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:13:46,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 53401f0681ce510d042c1599ea9bc7e2, store=B 2024-12-07T15:13:46,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:13:46,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 53401f0681ce510d042c1599ea9bc7e2, store=C 2024-12-07T15:13:46,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:13:46,861 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/A/cf8f68bb629d4426af4882827e61212d is 50, key is test_row_0/A:col10/1733584426180/Put/seqid=0 2024-12-07T15:13:46,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741870_1046 (size=12151) 2024-12-07T15:13:46,884 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/A/cf8f68bb629d4426af4882827e61212d 2024-12-07T15:13:46,915 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/B/e3f19ea1e55344999bce8c2d389200fa is 50, key is test_row_0/B:col10/1733584426180/Put/seqid=0 2024-12-07T15:13:46,932 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:46,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1733584486922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:46,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:46,933 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:46,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36040 deadline: 1733584486922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:46,933 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:46,933 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:46,934 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:46,934 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:46,934 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:46,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741871_1047 (size=12151) 2024-12-07T15:13:47,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:47,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1733584487044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:47,047 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:47,047 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:47,047 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:47,047 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:47,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36040 deadline: 1733584487044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:47,048 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:47,048 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:47,048 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:47,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:47,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1733584487253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:47,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:47,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:47,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:47,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:47,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36040 deadline: 1733584487254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:47,258 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:47,258 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:47,258 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:47,350 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/B/e3f19ea1e55344999bce8c2d389200fa 2024-12-07T15:13:47,365 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/C/a04be5ad74c64d208927c5b0b6ea8e79 is 50, key is test_row_0/C:col10/1733584426180/Put/seqid=0 2024-12-07T15:13:47,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741872_1048 (size=12151) 2024-12-07T15:13:47,382 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/C/a04be5ad74c64d208927c5b0b6ea8e79 2024-12-07T15:13:47,393 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/A/cf8f68bb629d4426af4882827e61212d as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/cf8f68bb629d4426af4882827e61212d 2024-12-07T15:13:47,401 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/cf8f68bb629d4426af4882827e61212d, entries=150, sequenceid=182, filesize=11.9 K 2024-12-07T15:13:47,402 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/B/e3f19ea1e55344999bce8c2d389200fa as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/e3f19ea1e55344999bce8c2d389200fa 2024-12-07T15:13:47,411 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/e3f19ea1e55344999bce8c2d389200fa, entries=150, sequenceid=182, filesize=11.9 K 2024-12-07T15:13:47,416 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/C/a04be5ad74c64d208927c5b0b6ea8e79 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/a04be5ad74c64d208927c5b0b6ea8e79 2024-12-07T15:13:47,423 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/a04be5ad74c64d208927c5b0b6ea8e79, entries=150, sequenceid=182, filesize=11.9 K 2024-12-07T15:13:47,425 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 53401f0681ce510d042c1599ea9bc7e2 in 588ms, sequenceid=182, compaction requested=true 2024-12-07T15:13:47,426 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 53401f0681ce510d042c1599ea9bc7e2: 2024-12-07T15:13:47,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 53401f0681ce510d042c1599ea9bc7e2:A, priority=-2147483648, current under compaction store size is 1 2024-12-07T15:13:47,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:13:47,426 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:13:47,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 53401f0681ce510d042c1599ea9bc7e2:B, priority=-2147483648, current under compaction store size is 2 2024-12-07T15:13:47,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:13:47,426 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:13:47,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 53401f0681ce510d042c1599ea9bc7e2:C, priority=-2147483648, current under compaction store size is 3 2024-12-07T15:13:47,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T15:13:47,427 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:13:47,427 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1541): 53401f0681ce510d042c1599ea9bc7e2/A is initiating minor compaction (all files) 2024-12-07T15:13:47,428 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 53401f0681ce510d042c1599ea9bc7e2/A in TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:47,428 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/89cd7191e60346948ebc9cd6b71d587a, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/bab8cfd5cb0949ac887ba3dafceb217e, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/cf8f68bb629d4426af4882827e61212d] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp, totalSize=35.9 K 2024-12-07T15:13:47,428 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:13:47,428 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1541): 53401f0681ce510d042c1599ea9bc7e2/B is initiating minor compaction (all files) 2024-12-07T15:13:47,428 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 53401f0681ce510d042c1599ea9bc7e2/B in TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:47,428 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89cd7191e60346948ebc9cd6b71d587a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1733584425251 2024-12-07T15:13:47,428 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/42d85a580a414589a2eb4a55b248f749, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/fc74713e99af49be9a1c3fb3f4b15798, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/e3f19ea1e55344999bce8c2d389200fa] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp, totalSize=35.9 K 2024-12-07T15:13:47,429 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting bab8cfd5cb0949ac887ba3dafceb217e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1733584425425 2024-12-07T15:13:47,429 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 42d85a580a414589a2eb4a55b248f749, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1733584425251 2024-12-07T15:13:47,429 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf8f68bb629d4426af4882827e61212d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1733584426180 2024-12-07T15:13:47,430 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting fc74713e99af49be9a1c3fb3f4b15798, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1733584425425 2024-12-07T15:13:47,431 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting e3f19ea1e55344999bce8c2d389200fa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1733584426180 2024-12-07T15:13:47,445 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 53401f0681ce510d042c1599ea9bc7e2#B#compaction#36 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:13:47,446 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/B/939c50a7356a485eabad8900a6142f61 is 50, key is test_row_0/B:col10/1733584426180/Put/seqid=0 2024-12-07T15:13:47,448 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 53401f0681ce510d042c1599ea9bc7e2#A#compaction#37 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:13:47,449 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/A/f0590943cc9b4ff88ac3631deecf9cb9 is 50, key is test_row_0/A:col10/1733584426180/Put/seqid=0 2024-12-07T15:13:47,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741873_1049 (size=12561) 2024-12-07T15:13:47,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741874_1050 (size=12561) 2024-12-07T15:13:47,464 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/B/939c50a7356a485eabad8900a6142f61 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/939c50a7356a485eabad8900a6142f61 2024-12-07T15:13:47,473 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 53401f0681ce510d042c1599ea9bc7e2/B of 53401f0681ce510d042c1599ea9bc7e2 into 939c50a7356a485eabad8900a6142f61(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:13:47,473 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 53401f0681ce510d042c1599ea9bc7e2: 2024-12-07T15:13:47,473 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., storeName=53401f0681ce510d042c1599ea9bc7e2/B, priority=13, startTime=1733584427426; duration=0sec 2024-12-07T15:13:47,473 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T15:13:47,473 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 53401f0681ce510d042c1599ea9bc7e2:B 2024-12-07T15:13:47,473 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:13:47,476 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:13:47,476 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1541): 53401f0681ce510d042c1599ea9bc7e2/C is initiating minor compaction (all files) 2024-12-07T15:13:47,476 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 53401f0681ce510d042c1599ea9bc7e2/C in TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:47,476 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/55b743c9a9064bd3950d11953a239329, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/b76cd182600f47e29d58b3b617198a8b, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/a04be5ad74c64d208927c5b0b6ea8e79] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp, totalSize=35.9 K 2024-12-07T15:13:47,477 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 55b743c9a9064bd3950d11953a239329, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1733584425251 2024-12-07T15:13:47,477 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting b76cd182600f47e29d58b3b617198a8b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1733584425425 2024-12-07T15:13:47,478 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting a04be5ad74c64d208927c5b0b6ea8e79, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1733584426180 2024-12-07T15:13:47,494 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 53401f0681ce510d042c1599ea9bc7e2#C#compaction#38 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:13:47,495 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/C/1658e68434e242f9bd559456ad36574d is 50, key is test_row_0/C:col10/1733584426180/Put/seqid=0 2024-12-07T15:13:47,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741875_1051 (size=12561) 2024-12-07T15:13:47,534 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/C/1658e68434e242f9bd559456ad36574d as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/1658e68434e242f9bd559456ad36574d 2024-12-07T15:13:47,547 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 53401f0681ce510d042c1599ea9bc7e2/C of 53401f0681ce510d042c1599ea9bc7e2 into 1658e68434e242f9bd559456ad36574d(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:13:47,547 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 53401f0681ce510d042c1599ea9bc7e2: 2024-12-07T15:13:47,547 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., storeName=53401f0681ce510d042c1599ea9bc7e2/C, priority=13, startTime=1733584427426; duration=0sec 2024-12-07T15:13:47,548 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:13:47,548 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 53401f0681ce510d042c1599ea9bc7e2:C 2024-12-07T15:13:47,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on 53401f0681ce510d042c1599ea9bc7e2 2024-12-07T15:13:47,568 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 53401f0681ce510d042c1599ea9bc7e2 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-07T15:13:47,568 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 53401f0681ce510d042c1599ea9bc7e2, store=A 2024-12-07T15:13:47,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:13:47,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 53401f0681ce510d042c1599ea9bc7e2, store=B 2024-12-07T15:13:47,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:13:47,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 53401f0681ce510d042c1599ea9bc7e2, store=C 2024-12-07T15:13:47,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:13:47,576 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/A/2dfa6e77cfd54c38af89ad1c4eea550a is 50, key is test_row_0/A:col10/1733584427566/Put/seqid=0 2024-12-07T15:13:47,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-07T15:13:47,594 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T15:13:47,595 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T15:13:47,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-12-07T15:13:47,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-07T15:13:47,598 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T15:13:47,600 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T15:13:47,601 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T15:13:47,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741876_1052 (size=12151) 2024-12-07T15:13:47,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:47,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1733584487611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:47,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:47,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:47,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:47,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:47,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36040 deadline: 1733584487611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:47,619 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:47,619 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:47,619 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:47,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-07T15:13:47,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:47,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1733584487724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:47,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:47,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36040 deadline: 1733584487724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:47,732 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:47,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:47,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:47,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:47,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:47,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:47,754 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-07T15:13:47,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:47,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. as already flushing 2024-12-07T15:13:47,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:47,754 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:13:47,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:13:47,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:13:47,874 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/A/f0590943cc9b4ff88ac3631deecf9cb9 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/f0590943cc9b4ff88ac3631deecf9cb9 2024-12-07T15:13:47,884 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 53401f0681ce510d042c1599ea9bc7e2/A of 53401f0681ce510d042c1599ea9bc7e2 into f0590943cc9b4ff88ac3631deecf9cb9(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:13:47,884 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 53401f0681ce510d042c1599ea9bc7e2: 2024-12-07T15:13:47,884 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., storeName=53401f0681ce510d042c1599ea9bc7e2/A, priority=13, startTime=1733584427426; duration=0sec 2024-12-07T15:13:47,884 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:13:47,884 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 53401f0681ce510d042c1599ea9bc7e2:A 2024-12-07T15:13:47,912 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-07T15:13:47,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:47,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. as already flushing 2024-12-07T15:13:47,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:47,913 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:13:47,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-07T15:13:47,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:13:47,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:13:47,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:47,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36040 deadline: 1733584487944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:47,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:47,946 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:47,946 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:47,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1733584487944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:47,946 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:47,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:47,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:47,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:48,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:48,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36068 deadline: 1733584488004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:48,008 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to d6797ebe9c52,46009,1733584418401 for 'test_row_0' in 53401f0681ce510d042c1599ea9bc7e2 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4228 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-07T15:13:48,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:48,010 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:48,010 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:48,010 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/A/2dfa6e77cfd54c38af89ad1c4eea550a 2024-12-07T15:13:48,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:48,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36024 deadline: 1733584488026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:48,031 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to d6797ebe9c52,46009,1733584418401 for 'test_row_0' in 53401f0681ce510d042c1599ea9bc7e2 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4228 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-07T15:13:48,032 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:48,032 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:48,032 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:48,034 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/B/1744017ec4b0452eb570b225b4016f57 is 50, key is test_row_0/B:col10/1733584427566/Put/seqid=0 2024-12-07T15:13:48,041 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:48,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36026 deadline: 1733584488034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:48,043 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to d6797ebe9c52,46009,1733584418401 for 'test_row_1' in 53401f0681ce510d042c1599ea9bc7e2 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4241 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-07T15:13:48,044 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:48,044 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:48,044 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:48,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741877_1053 (size=12151) 2024-12-07T15:13:48,067 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-07T15:13:48,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:48,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. as already flushing 2024-12-07T15:13:48,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:48,068 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:13:48,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:13:48,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:13:48,122 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-07T15:13:48,123 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-07T15:13:48,126 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-07T15:13:48,126 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-07T15:13:48,126 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-07T15:13:48,126 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-07T15:13:48,128 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-07T15:13:48,128 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-12-07T15:13:48,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-07T15:13:48,225 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-07T15:13:48,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:48,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. as already flushing 2024-12-07T15:13:48,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:48,226 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:13:48,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:13:48,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:13:48,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:48,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36040 deadline: 1733584488253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:48,263 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:48,263 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:48,263 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:48,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:48,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1733584488253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:48,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:48,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:48,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:48,387 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-07T15:13:48,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:48,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. as already flushing 2024-12-07T15:13:48,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:48,388 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:13:48,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:13:48,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:13:48,451 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/B/1744017ec4b0452eb570b225b4016f57 2024-12-07T15:13:48,479 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/C/e067375b409d4e38b8af5dc0b70a5670 is 50, key is test_row_0/C:col10/1733584427566/Put/seqid=0 2024-12-07T15:13:48,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741878_1054 (size=12151) 2024-12-07T15:13:48,544 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-07T15:13:48,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:48,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. as already flushing 2024-12-07T15:13:48,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:48,545 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:13:48,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:13:48,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:13:48,700 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-07T15:13:48,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:48,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. as already flushing 2024-12-07T15:13:48,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:48,701 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:13:48,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:13:48,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:13:48,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-07T15:13:48,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:48,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1733584488774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:48,781 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:48,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36040 deadline: 1733584488773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:48,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:48,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:48,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:48,781 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:48,781 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:48,781 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:48,855 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-07T15:13:48,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:48,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. as already flushing 2024-12-07T15:13:48,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:48,855 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:13:48,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:13:48,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:13:48,894 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/C/e067375b409d4e38b8af5dc0b70a5670 2024-12-07T15:13:48,904 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/A/2dfa6e77cfd54c38af89ad1c4eea550a as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/2dfa6e77cfd54c38af89ad1c4eea550a 2024-12-07T15:13:48,910 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/2dfa6e77cfd54c38af89ad1c4eea550a, entries=150, sequenceid=203, filesize=11.9 K 2024-12-07T15:13:48,914 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/B/1744017ec4b0452eb570b225b4016f57 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/1744017ec4b0452eb570b225b4016f57 2024-12-07T15:13:48,921 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/1744017ec4b0452eb570b225b4016f57, entries=150, sequenceid=203, filesize=11.9 K 2024-12-07T15:13:48,922 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/C/e067375b409d4e38b8af5dc0b70a5670 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/e067375b409d4e38b8af5dc0b70a5670 2024-12-07T15:13:48,934 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/e067375b409d4e38b8af5dc0b70a5670, entries=150, sequenceid=203, filesize=11.9 K 2024-12-07T15:13:48,936 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for 53401f0681ce510d042c1599ea9bc7e2 in 1368ms, sequenceid=203, compaction requested=false 2024-12-07T15:13:48,936 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 53401f0681ce510d042c1599ea9bc7e2: 2024-12-07T15:13:49,014 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-07T15:13:49,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:49,015 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 53401f0681ce510d042c1599ea9bc7e2 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-07T15:13:49,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 53401f0681ce510d042c1599ea9bc7e2, store=A 2024-12-07T15:13:49,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:13:49,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 53401f0681ce510d042c1599ea9bc7e2, store=B 2024-12-07T15:13:49,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:13:49,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 53401f0681ce510d042c1599ea9bc7e2, store=C 2024-12-07T15:13:49,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:13:49,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/A/1c71939fdabf454394af03e5acc43921 is 50, key is test_row_0/A:col10/1733584427607/Put/seqid=0 2024-12-07T15:13:49,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741879_1055 (size=12151) 2024-12-07T15:13:49,451 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/A/1c71939fdabf454394af03e5acc43921 2024-12-07T15:13:49,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/B/12634d9acca74457b3ebd2647f887899 is 50, key is test_row_0/B:col10/1733584427607/Put/seqid=0 2024-12-07T15:13:49,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741880_1056 (size=12151) 2024-12-07T15:13:49,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-07T15:13:49,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on 53401f0681ce510d042c1599ea9bc7e2 2024-12-07T15:13:49,796 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. as already flushing 2024-12-07T15:13:49,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:49,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1733584489860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:49,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:49,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:49,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:49,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:49,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36040 deadline: 1733584489866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:49,873 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:49,873 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:49,873 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:49,923 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/B/12634d9acca74457b3ebd2647f887899 2024-12-07T15:13:49,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/C/906c00c7eccd43dcb39afc58bbc5a183 is 50, key is test_row_0/C:col10/1733584427607/Put/seqid=0 2024-12-07T15:13:49,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741881_1057 (size=12151) 2024-12-07T15:13:49,977 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:49,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1733584489973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:49,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:49,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:49,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:49,984 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:49,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36040 deadline: 1733584489983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:49,984 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:49,984 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:49,984 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:50,186 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:50,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1733584490183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:50,187 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:50,187 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:50,187 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:50,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:50,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36040 deadline: 1733584490193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:50,198 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:50,198 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:50,198 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:50,342 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/C/906c00c7eccd43dcb39afc58bbc5a183 2024-12-07T15:13:50,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/A/1c71939fdabf454394af03e5acc43921 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/1c71939fdabf454394af03e5acc43921 2024-12-07T15:13:50,361 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/1c71939fdabf454394af03e5acc43921, entries=150, sequenceid=218, filesize=11.9 K 2024-12-07T15:13:50,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/B/12634d9acca74457b3ebd2647f887899 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/12634d9acca74457b3ebd2647f887899 2024-12-07T15:13:50,374 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/12634d9acca74457b3ebd2647f887899, entries=150, sequenceid=218, filesize=11.9 K 2024-12-07T15:13:50,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/C/906c00c7eccd43dcb39afc58bbc5a183 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/906c00c7eccd43dcb39afc58bbc5a183 2024-12-07T15:13:50,384 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/906c00c7eccd43dcb39afc58bbc5a183, entries=150, sequenceid=218, filesize=11.9 K 2024-12-07T15:13:50,386 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 53401f0681ce510d042c1599ea9bc7e2 in 1371ms, sequenceid=218, compaction requested=true 2024-12-07T15:13:50,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 53401f0681ce510d042c1599ea9bc7e2: 2024-12-07T15:13:50,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:50,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-12-07T15:13:50,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-12-07T15:13:50,392 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-12-07T15:13:50,392 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7880 sec 2024-12-07T15:13:50,394 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 2.7970 sec 2024-12-07T15:13:50,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on 53401f0681ce510d042c1599ea9bc7e2 2024-12-07T15:13:50,497 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 53401f0681ce510d042c1599ea9bc7e2 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-07T15:13:50,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 53401f0681ce510d042c1599ea9bc7e2, store=A 2024-12-07T15:13:50,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:13:50,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 53401f0681ce510d042c1599ea9bc7e2, store=B 2024-12-07T15:13:50,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:13:50,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 53401f0681ce510d042c1599ea9bc7e2, store=C 2024-12-07T15:13:50,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:13:50,506 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/A/9859a436fdc54dbe94857cbbac4ce62a is 50, key is test_row_0/A:col10/1733584430495/Put/seqid=0 2024-12-07T15:13:50,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741882_1058 (size=14541) 2024-12-07T15:13:50,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:50,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36040 deadline: 1733584490563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:50,568 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:50,569 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:50,569 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:50,573 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:50,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1733584490568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:50,574 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:50,574 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:50,574 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:50,676 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:50,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36040 deadline: 1733584490673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:50,677 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:50,677 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:50,677 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:50,687 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:50,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1733584490683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:50,688 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:50,688 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:50,688 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:50,884 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:50,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36040 deadline: 1733584490883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:50,885 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:50,885 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:50,885 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:50,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:50,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1733584490893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:50,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:50,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:50,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:50,923 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/A/9859a436fdc54dbe94857cbbac4ce62a 2024-12-07T15:13:50,934 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/B/614e2024813945019a2586f3551f080b is 50, key is test_row_0/B:col10/1733584430495/Put/seqid=0 2024-12-07T15:13:50,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741883_1059 (size=12151) 2024-12-07T15:13:50,940 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/B/614e2024813945019a2586f3551f080b 2024-12-07T15:13:50,951 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/C/321d9470b4c54af391a5497ce8d5a3df is 50, key is test_row_0/C:col10/1733584430495/Put/seqid=0 2024-12-07T15:13:50,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741884_1060 (size=12151) 2024-12-07T15:13:51,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:51,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36040 deadline: 1733584491193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:51,200 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:51,200 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:51,200 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:51,217 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:51,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1733584491213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:51,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:51,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:51,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:51,357 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/C/321d9470b4c54af391a5497ce8d5a3df 2024-12-07T15:13:51,365 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/A/9859a436fdc54dbe94857cbbac4ce62a as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/9859a436fdc54dbe94857cbbac4ce62a 2024-12-07T15:13:51,370 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/9859a436fdc54dbe94857cbbac4ce62a, entries=200, sequenceid=237, filesize=14.2 K 2024-12-07T15:13:51,371 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/B/614e2024813945019a2586f3551f080b as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/614e2024813945019a2586f3551f080b 2024-12-07T15:13:51,377 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/614e2024813945019a2586f3551f080b, entries=150, sequenceid=237, filesize=11.9 K 2024-12-07T15:13:51,378 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/C/321d9470b4c54af391a5497ce8d5a3df as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/321d9470b4c54af391a5497ce8d5a3df 2024-12-07T15:13:51,384 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/321d9470b4c54af391a5497ce8d5a3df, entries=150, sequenceid=237, filesize=11.9 K 2024-12-07T15:13:51,385 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 53401f0681ce510d042c1599ea9bc7e2 in 888ms, sequenceid=237, compaction requested=true 2024-12-07T15:13:51,385 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 53401f0681ce510d042c1599ea9bc7e2: 2024-12-07T15:13:51,385 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 53401f0681ce510d042c1599ea9bc7e2:A, priority=-2147483648, current under compaction store size is 1 2024-12-07T15:13:51,385 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:13:51,385 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 53401f0681ce510d042c1599ea9bc7e2:B, priority=-2147483648, current under compaction store size is 2 2024-12-07T15:13:51,385 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-07T15:13:51,385 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:13:51,385 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-07T15:13:51,385 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 53401f0681ce510d042c1599ea9bc7e2:C, priority=-2147483648, current under compaction store size is 3 2024-12-07T15:13:51,385 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T15:13:51,387 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-07T15:13:51,387 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51404 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-07T15:13:51,387 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1541): 53401f0681ce510d042c1599ea9bc7e2/B is initiating minor compaction (all files) 2024-12-07T15:13:51,387 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1541): 53401f0681ce510d042c1599ea9bc7e2/A is initiating minor compaction (all files) 2024-12-07T15:13:51,387 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 53401f0681ce510d042c1599ea9bc7e2/B in TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:51,387 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 53401f0681ce510d042c1599ea9bc7e2/A in TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:51,387 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/939c50a7356a485eabad8900a6142f61, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/1744017ec4b0452eb570b225b4016f57, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/12634d9acca74457b3ebd2647f887899, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/614e2024813945019a2586f3551f080b] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp, totalSize=47.9 K 2024-12-07T15:13:51,387 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/f0590943cc9b4ff88ac3631deecf9cb9, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/2dfa6e77cfd54c38af89ad1c4eea550a, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/1c71939fdabf454394af03e5acc43921, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/9859a436fdc54dbe94857cbbac4ce62a] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp, totalSize=50.2 K 2024-12-07T15:13:51,388 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 939c50a7356a485eabad8900a6142f61, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1733584426180 2024-12-07T15:13:51,388 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting f0590943cc9b4ff88ac3631deecf9cb9, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1733584426180 2024-12-07T15:13:51,388 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 1744017ec4b0452eb570b225b4016f57, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1733584426890 2024-12-07T15:13:51,389 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2dfa6e77cfd54c38af89ad1c4eea550a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1733584426890 2024-12-07T15:13:51,389 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 12634d9acca74457b3ebd2647f887899, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1733584427580 2024-12-07T15:13:51,389 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c71939fdabf454394af03e5acc43921, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1733584427580 2024-12-07T15:13:51,389 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 614e2024813945019a2586f3551f080b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1733584429855 2024-12-07T15:13:51,390 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9859a436fdc54dbe94857cbbac4ce62a, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1733584429855 2024-12-07T15:13:51,404 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 53401f0681ce510d042c1599ea9bc7e2#B#compaction#48 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:13:51,405 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/B/b1399a8d338046ce988ea72cfb3ab3e8 is 50, key is test_row_0/B:col10/1733584430495/Put/seqid=0 2024-12-07T15:13:51,406 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 53401f0681ce510d042c1599ea9bc7e2#A#compaction#49 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:13:51,406 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/A/28b9003d663242d1bea9c6e7d20b3f99 is 50, key is test_row_0/A:col10/1733584430495/Put/seqid=0 2024-12-07T15:13:51,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741885_1061 (size=12697) 2024-12-07T15:13:51,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741886_1062 (size=12697) 2024-12-07T15:13:51,436 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/A/28b9003d663242d1bea9c6e7d20b3f99 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/28b9003d663242d1bea9c6e7d20b3f99 2024-12-07T15:13:51,448 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 53401f0681ce510d042c1599ea9bc7e2/A of 53401f0681ce510d042c1599ea9bc7e2 into 28b9003d663242d1bea9c6e7d20b3f99(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:13:51,448 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 53401f0681ce510d042c1599ea9bc7e2: 2024-12-07T15:13:51,448 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., storeName=53401f0681ce510d042c1599ea9bc7e2/A, priority=12, startTime=1733584431385; duration=0sec 2024-12-07T15:13:51,448 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T15:13:51,448 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 53401f0681ce510d042c1599ea9bc7e2:A 2024-12-07T15:13:51,448 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-07T15:13:51,451 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-07T15:13:51,451 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1541): 53401f0681ce510d042c1599ea9bc7e2/C is initiating minor compaction (all files) 2024-12-07T15:13:51,451 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 53401f0681ce510d042c1599ea9bc7e2/C in TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:51,451 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/1658e68434e242f9bd559456ad36574d, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/e067375b409d4e38b8af5dc0b70a5670, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/906c00c7eccd43dcb39afc58bbc5a183, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/321d9470b4c54af391a5497ce8d5a3df] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp, totalSize=47.9 K 2024-12-07T15:13:51,452 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1658e68434e242f9bd559456ad36574d, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1733584426180 2024-12-07T15:13:51,452 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting e067375b409d4e38b8af5dc0b70a5670, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1733584426890 2024-12-07T15:13:51,452 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 906c00c7eccd43dcb39afc58bbc5a183, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1733584427580 2024-12-07T15:13:51,453 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 321d9470b4c54af391a5497ce8d5a3df, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1733584429855 2024-12-07T15:13:51,468 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 53401f0681ce510d042c1599ea9bc7e2#C#compaction#50 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:13:51,469 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/C/de3f5403c83b4aec8338ea7d0c3fa7e2 is 50, key is test_row_0/C:col10/1733584430495/Put/seqid=0 2024-12-07T15:13:51,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741887_1063 (size=12697) 2024-12-07T15:13:51,718 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 53401f0681ce510d042c1599ea9bc7e2 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-07T15:13:51,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 53401f0681ce510d042c1599ea9bc7e2, store=A 2024-12-07T15:13:51,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:13:51,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 53401f0681ce510d042c1599ea9bc7e2, store=B 2024-12-07T15:13:51,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:13:51,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 53401f0681ce510d042c1599ea9bc7e2, store=C 2024-12-07T15:13:51,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:13:51,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on 53401f0681ce510d042c1599ea9bc7e2 2024-12-07T15:13:51,731 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/A/4f37ced65c3e40dd85afb3c068a8a50a is 50, key is test_row_0/A:col10/1733584431716/Put/seqid=0 2024-12-07T15:13:51,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741888_1064 (size=14541) 2024-12-07T15:13:51,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-07T15:13:51,763 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T15:13:51,764 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T15:13:51,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-12-07T15:13:51,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-07T15:13:51,768 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T15:13:51,770 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T15:13:51,770 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T15:13:51,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:51,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1733584491779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:51,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:51,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:51,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:51,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:51,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36040 deadline: 1733584491781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:51,787 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:51,787 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:51,787 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:51,826 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/B/b1399a8d338046ce988ea72cfb3ab3e8 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/b1399a8d338046ce988ea72cfb3ab3e8 2024-12-07T15:13:51,837 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 53401f0681ce510d042c1599ea9bc7e2/B of 53401f0681ce510d042c1599ea9bc7e2 into b1399a8d338046ce988ea72cfb3ab3e8(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:13:51,837 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 53401f0681ce510d042c1599ea9bc7e2: 2024-12-07T15:13:51,837 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., storeName=53401f0681ce510d042c1599ea9bc7e2/B, priority=12, startTime=1733584431385; duration=0sec 2024-12-07T15:13:51,837 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:13:51,837 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 53401f0681ce510d042c1599ea9bc7e2:B 2024-12-07T15:13:51,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-07T15:13:51,886 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/C/de3f5403c83b4aec8338ea7d0c3fa7e2 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/de3f5403c83b4aec8338ea7d0c3fa7e2 2024-12-07T15:13:51,895 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 53401f0681ce510d042c1599ea9bc7e2/C of 53401f0681ce510d042c1599ea9bc7e2 into de3f5403c83b4aec8338ea7d0c3fa7e2(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:13:51,895 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 53401f0681ce510d042c1599ea9bc7e2: 2024-12-07T15:13:51,895 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., storeName=53401f0681ce510d042c1599ea9bc7e2/C, priority=12, startTime=1733584431385; duration=0sec 2024-12-07T15:13:51,895 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:13:51,895 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 53401f0681ce510d042c1599ea9bc7e2:C 2024-12-07T15:13:51,896 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:51,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1733584491893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:51,897 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:51,897 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:51,897 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:51,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:51,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36040 deadline: 1733584491894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:51,897 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:51,898 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:51,898 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:51,926 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-07T15:13:51,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:51,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. as already flushing 2024-12-07T15:13:51,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:51,927 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:13:51,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:13:51,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:13:52,051 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:52,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36068 deadline: 1733584492043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:52,052 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to d6797ebe9c52,46009,1733584418401 for 'test_row_0' in 53401f0681ce510d042c1599ea9bc7e2 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8270 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-07T15:13:52,053 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:52,053 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:52,053 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:52,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:52,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36024 deadline: 1733584492043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:52,054 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to d6797ebe9c52,46009,1733584418401 for 'test_row_0' in 53401f0681ce510d042c1599ea9bc7e2 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8252 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-07T15:13:52,055 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:52,055 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:52,055 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:52,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:52,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36026 deadline: 1733584492063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:52,067 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to d6797ebe9c52,46009,1733584418401 for 'test_row_1' in 53401f0681ce510d042c1599ea9bc7e2 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8264 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-07T15:13:52,067 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:52,067 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:52,067 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:52,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-07T15:13:52,086 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-07T15:13:52,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:52,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. as already flushing 2024-12-07T15:13:52,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:52,086 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:13:52,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:13:52,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:13:52,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:52,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1733584492104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:52,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:52,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:52,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:52,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:52,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36040 deadline: 1733584492104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:52,106 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:52,106 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:52,106 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:52,151 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/A/4f37ced65c3e40dd85afb3c068a8a50a 2024-12-07T15:13:52,163 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/B/17528d69805e4b7bac291b5b269acfc2 is 50, key is test_row_0/B:col10/1733584431716/Put/seqid=0 2024-12-07T15:13:52,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741889_1065 (size=12151) 2024-12-07T15:13:52,173 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/B/17528d69805e4b7bac291b5b269acfc2 2024-12-07T15:13:52,183 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/C/c7b6b1f11237452092223227307bb527 is 50, key is test_row_0/C:col10/1733584431716/Put/seqid=0 2024-12-07T15:13:52,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741890_1066 (size=12151) 2024-12-07T15:13:52,240 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-07T15:13:52,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:52,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. as already flushing 2024-12-07T15:13:52,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:52,241 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:13:52,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:13:52,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:13:52,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-07T15:13:52,395 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-07T15:13:52,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:52,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. as already flushing 2024-12-07T15:13:52,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:52,395 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:13:52,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:13:52,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:13:52,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:52,414 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:52,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1733584492413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:52,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36040 deadline: 1733584492414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:52,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:52,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:52,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:52,422 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:52,422 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:52,422 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:52,550 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-07T15:13:52,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:52,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. as already flushing 2024-12-07T15:13:52,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:52,551 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:13:52,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:13:52,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:13:52,596 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/C/c7b6b1f11237452092223227307bb527 2024-12-07T15:13:52,603 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/A/4f37ced65c3e40dd85afb3c068a8a50a as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/4f37ced65c3e40dd85afb3c068a8a50a 2024-12-07T15:13:52,614 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/4f37ced65c3e40dd85afb3c068a8a50a, entries=200, sequenceid=252, filesize=14.2 K 2024-12-07T15:13:52,615 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/B/17528d69805e4b7bac291b5b269acfc2 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/17528d69805e4b7bac291b5b269acfc2 2024-12-07T15:13:52,621 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/17528d69805e4b7bac291b5b269acfc2, entries=150, sequenceid=252, filesize=11.9 K 2024-12-07T15:13:52,622 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/C/c7b6b1f11237452092223227307bb527 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/c7b6b1f11237452092223227307bb527 2024-12-07T15:13:52,629 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/c7b6b1f11237452092223227307bb527, entries=150, sequenceid=252, filesize=11.9 K 2024-12-07T15:13:52,633 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 53401f0681ce510d042c1599ea9bc7e2 in 915ms, sequenceid=252, compaction requested=false 2024-12-07T15:13:52,633 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 53401f0681ce510d042c1599ea9bc7e2: 2024-12-07T15:13:52,704 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-07T15:13:52,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:52,705 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 53401f0681ce510d042c1599ea9bc7e2 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-07T15:13:52,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 53401f0681ce510d042c1599ea9bc7e2, store=A 2024-12-07T15:13:52,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:13:52,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 53401f0681ce510d042c1599ea9bc7e2, store=B 2024-12-07T15:13:52,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:13:52,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 53401f0681ce510d042c1599ea9bc7e2, store=C 2024-12-07T15:13:52,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:13:52,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/A/7ca55ff415ce4f1cb587a3a6484025de is 50, key is test_row_0/A:col10/1733584431770/Put/seqid=0 2024-12-07T15:13:52,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741891_1067 (size=12301) 2024-12-07T15:13:52,718 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/A/7ca55ff415ce4f1cb587a3a6484025de 2024-12-07T15:13:52,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/B/bb9533654c7c49c9809659effedd3a3a is 50, key is test_row_0/B:col10/1733584431770/Put/seqid=0 2024-12-07T15:13:52,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741892_1068 (size=12301) 2024-12-07T15:13:52,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-07T15:13:52,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on 53401f0681ce510d042c1599ea9bc7e2 2024-12-07T15:13:52,935 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. as already flushing 2024-12-07T15:13:53,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:53,009 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:53,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36040 deadline: 1733584493004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:53,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1733584493004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:53,010 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:53,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:53,010 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:53,010 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:53,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:53,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:53,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:53,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36040 deadline: 1733584493114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:53,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:53,115 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:53,116 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:53,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1733584493114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:53,116 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:53,116 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:53,116 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:53,116 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:53,135 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/B/bb9533654c7c49c9809659effedd3a3a 2024-12-07T15:13:53,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/C/7b446d9de80c4b18bba71d6996067c8b is 50, key is test_row_0/C:col10/1733584431770/Put/seqid=0 2024-12-07T15:13:53,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741893_1069 (size=12301) 2024-12-07T15:13:53,187 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/C/7b446d9de80c4b18bba71d6996067c8b 2024-12-07T15:13:53,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/A/7ca55ff415ce4f1cb587a3a6484025de as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/7ca55ff415ce4f1cb587a3a6484025de 2024-12-07T15:13:53,207 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/7ca55ff415ce4f1cb587a3a6484025de, entries=150, sequenceid=272, filesize=12.0 K 2024-12-07T15:13:53,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/B/bb9533654c7c49c9809659effedd3a3a as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/bb9533654c7c49c9809659effedd3a3a 2024-12-07T15:13:53,220 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/bb9533654c7c49c9809659effedd3a3a, entries=150, sequenceid=272, filesize=12.0 K 2024-12-07T15:13:53,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/C/7b446d9de80c4b18bba71d6996067c8b as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/7b446d9de80c4b18bba71d6996067c8b 2024-12-07T15:13:53,239 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/7b446d9de80c4b18bba71d6996067c8b, entries=150, sequenceid=272, filesize=12.0 K 2024-12-07T15:13:53,241 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for 53401f0681ce510d042c1599ea9bc7e2 in 536ms, sequenceid=272, compaction requested=true 2024-12-07T15:13:53,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 53401f0681ce510d042c1599ea9bc7e2: 2024-12-07T15:13:53,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:53,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-12-07T15:13:53,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-12-07T15:13:53,247 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-12-07T15:13:53,247 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4730 sec 2024-12-07T15:13:53,250 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.4840 sec 2024-12-07T15:13:53,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on 53401f0681ce510d042c1599ea9bc7e2 2024-12-07T15:13:53,336 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 53401f0681ce510d042c1599ea9bc7e2 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-07T15:13:53,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 53401f0681ce510d042c1599ea9bc7e2, store=A 2024-12-07T15:13:53,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:13:53,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 53401f0681ce510d042c1599ea9bc7e2, store=B 2024-12-07T15:13:53,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:13:53,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 53401f0681ce510d042c1599ea9bc7e2, store=C 2024-12-07T15:13:53,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:13:53,346 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/A/6916c638b3d74ed8941c157beefb47f9 is 50, key is test_row_0/A:col10/1733584433334/Put/seqid=0 2024-12-07T15:13:53,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741894_1070 (size=12301) 2024-12-07T15:13:53,384 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:53,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1733584493373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:53,386 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:53,386 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:53,386 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:53,386 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:53,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36040 deadline: 1733584493385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:53,386 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:53,386 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:53,387 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:53,496 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:53,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1733584493493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:53,497 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:53,497 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:53,497 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:53,497 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:53,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36040 deadline: 1733584493493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:53,498 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:53,498 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:53,498 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:53,706 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:53,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1733584493703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:53,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:53,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:53,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:53,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:53,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36040 deadline: 1733584493703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:53,708 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:53,708 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:53,708 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:53,763 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/A/6916c638b3d74ed8941c157beefb47f9 2024-12-07T15:13:53,778 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/B/76302b6cebb845269b72194877e1f507 is 50, key is test_row_0/B:col10/1733584433334/Put/seqid=0 2024-12-07T15:13:53,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741895_1071 (size=12301) 2024-12-07T15:13:53,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-07T15:13:53,914 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T15:13:53,914 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T15:13:53,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-12-07T15:13:53,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-07T15:13:53,917 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T15:13:53,919 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T15:13:53,919 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T15:13:54,014 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:54,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36040 deadline: 1733584494014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:54,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:54,015 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:54,015 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:54,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1733584494014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:54,015 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:54,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:54,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:54,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:54,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-07T15:13:54,072 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-07T15:13:54,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:54,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. as already flushing 2024-12-07T15:13:54,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:54,072 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:13:54,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:13:54,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:13:54,191 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/B/76302b6cebb845269b72194877e1f507 2024-12-07T15:13:54,204 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/C/d99cbc6ca10b45e7a1129c2097b33c8d is 50, key is test_row_0/C:col10/1733584433334/Put/seqid=0 2024-12-07T15:13:54,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741896_1072 (size=12301) 2024-12-07T15:13:54,226 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-07T15:13:54,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:54,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. as already flushing 2024-12-07T15:13:54,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:54,227 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:13:54,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:13:54,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:13:54,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-07T15:13:54,380 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-07T15:13:54,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:54,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. as already flushing 2024-12-07T15:13:54,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:54,380 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:13:54,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:13:54,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:13:54,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:54,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36040 deadline: 1733584494523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:54,524 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:54,524 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:54,525 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:54,534 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-07T15:13:54,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:54,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. as already flushing 2024-12-07T15:13:54,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:54,534 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:13:54,534 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:54,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:13:54,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1733584494533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:54,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:54,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:54,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:54,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:13:54,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-07T15:13:54,609 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/C/d99cbc6ca10b45e7a1129c2097b33c8d 2024-12-07T15:13:54,616 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/A/6916c638b3d74ed8941c157beefb47f9 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/6916c638b3d74ed8941c157beefb47f9 2024-12-07T15:13:54,622 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/6916c638b3d74ed8941c157beefb47f9, entries=150, sequenceid=289, filesize=12.0 K 2024-12-07T15:13:54,624 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/B/76302b6cebb845269b72194877e1f507 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/76302b6cebb845269b72194877e1f507 2024-12-07T15:13:54,631 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/76302b6cebb845269b72194877e1f507, entries=150, sequenceid=289, filesize=12.0 K 2024-12-07T15:13:54,632 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/C/d99cbc6ca10b45e7a1129c2097b33c8d as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/d99cbc6ca10b45e7a1129c2097b33c8d 2024-12-07T15:13:54,641 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/d99cbc6ca10b45e7a1129c2097b33c8d, entries=150, sequenceid=289, filesize=12.0 K 2024-12-07T15:13:54,643 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 53401f0681ce510d042c1599ea9bc7e2 in 1307ms, sequenceid=289, compaction requested=true 2024-12-07T15:13:54,643 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 53401f0681ce510d042c1599ea9bc7e2: 2024-12-07T15:13:54,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 53401f0681ce510d042c1599ea9bc7e2:A, priority=-2147483648, current under compaction store size is 1 2024-12-07T15:13:54,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:13:54,643 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-07T15:13:54,643 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-07T15:13:54,644 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 53401f0681ce510d042c1599ea9bc7e2:B, priority=-2147483648, current under compaction store size is 2 2024-12-07T15:13:54,644 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:13:54,644 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 53401f0681ce510d042c1599ea9bc7e2:C, priority=-2147483648, current under compaction store size is 3 2024-12-07T15:13:54,644 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T15:13:54,646 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49450 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-07T15:13:54,646 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1541): 53401f0681ce510d042c1599ea9bc7e2/B is initiating minor compaction (all files) 2024-12-07T15:13:54,646 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 53401f0681ce510d042c1599ea9bc7e2/B in TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:54,646 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/b1399a8d338046ce988ea72cfb3ab3e8, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/17528d69805e4b7bac291b5b269acfc2, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/bb9533654c7c49c9809659effedd3a3a, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/76302b6cebb845269b72194877e1f507] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp, totalSize=48.3 K 2024-12-07T15:13:54,647 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51840 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-07T15:13:54,647 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1541): 53401f0681ce510d042c1599ea9bc7e2/A is initiating minor compaction (all files) 2024-12-07T15:13:54,647 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 53401f0681ce510d042c1599ea9bc7e2/A in TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:54,647 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/28b9003d663242d1bea9c6e7d20b3f99, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/4f37ced65c3e40dd85afb3c068a8a50a, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/7ca55ff415ce4f1cb587a3a6484025de, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/6916c638b3d74ed8941c157beefb47f9] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp, totalSize=50.6 K 2024-12-07T15:13:54,648 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting b1399a8d338046ce988ea72cfb3ab3e8, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1733584429855 2024-12-07T15:13:54,649 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 28b9003d663242d1bea9c6e7d20b3f99, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1733584429855 2024-12-07T15:13:54,649 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 17528d69805e4b7bac291b5b269acfc2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1733584430506 2024-12-07T15:13:54,649 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4f37ced65c3e40dd85afb3c068a8a50a, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1733584430506 2024-12-07T15:13:54,652 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting bb9533654c7c49c9809659effedd3a3a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1733584431770 2024-12-07T15:13:54,652 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ca55ff415ce4f1cb587a3a6484025de, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1733584431770 2024-12-07T15:13:54,653 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 76302b6cebb845269b72194877e1f507, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1733584432969 2024-12-07T15:13:54,653 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6916c638b3d74ed8941c157beefb47f9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1733584432969 2024-12-07T15:13:54,671 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 53401f0681ce510d042c1599ea9bc7e2#B#compaction#60 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:13:54,672 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/B/dd640f8ad6d9483cad6b4bdad1827b9b is 50, key is test_row_0/B:col10/1733584433334/Put/seqid=0 2024-12-07T15:13:54,677 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 53401f0681ce510d042c1599ea9bc7e2#A#compaction#61 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:13:54,678 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/A/9dc7ea62c09f426d931bc23d362be474 is 50, key is test_row_0/A:col10/1733584433334/Put/seqid=0 2024-12-07T15:13:54,688 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-07T15:13:54,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:54,688 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 53401f0681ce510d042c1599ea9bc7e2 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-07T15:13:54,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 53401f0681ce510d042c1599ea9bc7e2, store=A 2024-12-07T15:13:54,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:13:54,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 53401f0681ce510d042c1599ea9bc7e2, store=B 2024-12-07T15:13:54,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:13:54,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 53401f0681ce510d042c1599ea9bc7e2, store=C 2024-12-07T15:13:54,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:13:54,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741897_1073 (size=12983) 2024-12-07T15:13:54,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/A/eacf947c67264a109c5ccceb000c3bca is 50, key is test_row_0/A:col10/1733584433383/Put/seqid=0 2024-12-07T15:13:54,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741898_1074 (size=12983) 2024-12-07T15:13:54,704 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/B/dd640f8ad6d9483cad6b4bdad1827b9b as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/dd640f8ad6d9483cad6b4bdad1827b9b 2024-12-07T15:13:54,712 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/A/9dc7ea62c09f426d931bc23d362be474 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/9dc7ea62c09f426d931bc23d362be474 2024-12-07T15:13:54,717 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 53401f0681ce510d042c1599ea9bc7e2/B of 53401f0681ce510d042c1599ea9bc7e2 into dd640f8ad6d9483cad6b4bdad1827b9b(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:13:54,717 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 53401f0681ce510d042c1599ea9bc7e2: 2024-12-07T15:13:54,717 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., storeName=53401f0681ce510d042c1599ea9bc7e2/B, priority=12, startTime=1733584434643; duration=0sec 2024-12-07T15:13:54,718 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T15:13:54,718 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 53401f0681ce510d042c1599ea9bc7e2:B 2024-12-07T15:13:54,718 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-07T15:13:54,722 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49450 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-07T15:13:54,722 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1541): 53401f0681ce510d042c1599ea9bc7e2/C is initiating minor compaction (all files) 2024-12-07T15:13:54,723 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 53401f0681ce510d042c1599ea9bc7e2/C in TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:54,723 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/de3f5403c83b4aec8338ea7d0c3fa7e2, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/c7b6b1f11237452092223227307bb527, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/7b446d9de80c4b18bba71d6996067c8b, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/d99cbc6ca10b45e7a1129c2097b33c8d] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp, totalSize=48.3 K 2024-12-07T15:13:54,723 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting de3f5403c83b4aec8338ea7d0c3fa7e2, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1733584429855 2024-12-07T15:13:54,724 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting c7b6b1f11237452092223227307bb527, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1733584430506 2024-12-07T15:13:54,725 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b446d9de80c4b18bba71d6996067c8b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1733584431770 2024-12-07T15:13:54,725 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting d99cbc6ca10b45e7a1129c2097b33c8d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1733584432969 2024-12-07T15:13:54,726 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 53401f0681ce510d042c1599ea9bc7e2/A of 53401f0681ce510d042c1599ea9bc7e2 into 9dc7ea62c09f426d931bc23d362be474(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:13:54,727 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 53401f0681ce510d042c1599ea9bc7e2: 2024-12-07T15:13:54,727 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., storeName=53401f0681ce510d042c1599ea9bc7e2/A, priority=12, startTime=1733584434643; duration=0sec 2024-12-07T15:13:54,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741899_1075 (size=12301) 2024-12-07T15:13:54,727 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:13:54,727 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 53401f0681ce510d042c1599ea9bc7e2:A 2024-12-07T15:13:54,728 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/A/eacf947c67264a109c5ccceb000c3bca 2024-12-07T15:13:54,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/B/2795874958f646d69ab7fe486e2e969c is 50, key is test_row_0/B:col10/1733584433383/Put/seqid=0 2024-12-07T15:13:54,751 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 53401f0681ce510d042c1599ea9bc7e2#C#compaction#64 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:13:54,760 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/C/87d27ff4e6764394b111eee75e3daf8a is 50, key is test_row_0/C:col10/1733584433334/Put/seqid=0 2024-12-07T15:13:54,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741900_1076 (size=12301) 2024-12-07T15:13:54,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741901_1077 (size=12983) 2024-12-07T15:13:55,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-07T15:13:55,176 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/B/2795874958f646d69ab7fe486e2e969c 2024-12-07T15:13:55,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/C/c0f918b435074fe3b366dd0923e81539 is 50, key is test_row_0/C:col10/1733584433383/Put/seqid=0 2024-12-07T15:13:55,193 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/C/87d27ff4e6764394b111eee75e3daf8a as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/87d27ff4e6764394b111eee75e3daf8a 2024-12-07T15:13:55,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741902_1078 (size=12301) 2024-12-07T15:13:55,203 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 53401f0681ce510d042c1599ea9bc7e2/C of 53401f0681ce510d042c1599ea9bc7e2 into 87d27ff4e6764394b111eee75e3daf8a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:13:55,203 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 53401f0681ce510d042c1599ea9bc7e2: 2024-12-07T15:13:55,203 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., storeName=53401f0681ce510d042c1599ea9bc7e2/C, priority=12, startTime=1733584434644; duration=0sec 2024-12-07T15:13:55,203 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:13:55,203 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 53401f0681ce510d042c1599ea9bc7e2:C 2024-12-07T15:13:55,204 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/C/c0f918b435074fe3b366dd0923e81539 2024-12-07T15:13:55,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/A/eacf947c67264a109c5ccceb000c3bca as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/eacf947c67264a109c5ccceb000c3bca 2024-12-07T15:13:55,217 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/eacf947c67264a109c5ccceb000c3bca, entries=150, sequenceid=304, filesize=12.0 K 2024-12-07T15:13:55,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/B/2795874958f646d69ab7fe486e2e969c as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/2795874958f646d69ab7fe486e2e969c 2024-12-07T15:13:55,227 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/2795874958f646d69ab7fe486e2e969c, entries=150, sequenceid=304, filesize=12.0 K 2024-12-07T15:13:55,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/C/c0f918b435074fe3b366dd0923e81539 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/c0f918b435074fe3b366dd0923e81539 2024-12-07T15:13:55,234 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/c0f918b435074fe3b366dd0923e81539, entries=150, sequenceid=304, filesize=12.0 K 2024-12-07T15:13:55,236 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=0 B/0 for 53401f0681ce510d042c1599ea9bc7e2 in 548ms, sequenceid=304, compaction requested=false 2024-12-07T15:13:55,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 53401f0681ce510d042c1599ea9bc7e2: 2024-12-07T15:13:55,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:55,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-12-07T15:13:55,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-12-07T15:13:55,242 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-12-07T15:13:55,242 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3200 sec 2024-12-07T15:13:55,244 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.3280 sec 2024-12-07T15:13:55,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on 53401f0681ce510d042c1599ea9bc7e2 2024-12-07T15:13:55,555 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 53401f0681ce510d042c1599ea9bc7e2 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-07T15:13:55,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 53401f0681ce510d042c1599ea9bc7e2, store=A 2024-12-07T15:13:55,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:13:55,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 53401f0681ce510d042c1599ea9bc7e2, store=B 2024-12-07T15:13:55,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:13:55,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 53401f0681ce510d042c1599ea9bc7e2, store=C 2024-12-07T15:13:55,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:13:55,570 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/A/4a12ef7335bc44a1873dd4f8b79e2e22 is 50, key is test_row_0/A:col10/1733584435551/Put/seqid=0 2024-12-07T15:13:55,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741903_1079 (size=17181) 2024-12-07T15:13:55,606 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/A/4a12ef7335bc44a1873dd4f8b79e2e22 2024-12-07T15:13:55,618 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/B/7d2e9e37dc504eeb9565c8ad6bb9ef25 is 50, key is test_row_0/B:col10/1733584435551/Put/seqid=0 2024-12-07T15:13:55,622 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733584435599, max timestamp=1733584435610Num uniques 859; 2024-12-07T15:13:55,622 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-07T15:13:55,624 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733584435599, max timestamp=1733584435610Num uniques 859; 2024-12-07T15:13:55,624 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-07T15:13:55,624 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733584435599, max timestamp=1733584435610Num uniques 859; 2024-12-07T15:13:55,625 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-07T15:13:55,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741904_1080 (size=12301) 2024-12-07T15:13:55,680 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:55,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1733584495675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:55,681 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:55,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:55,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:55,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36040 deadline: 1733584495679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:55,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:55,682 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:55,682 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:55,682 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:55,787 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:55,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1733584495783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:55,788 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:55,788 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:55,788 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:55,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:55,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36040 deadline: 1733584495794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:55,799 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:55,799 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:55,799 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:55,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:55,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1733584495993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:55,997 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:55,997 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:55,997 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:56,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:56,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36040 deadline: 1733584496004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:56,009 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:56,009 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:56,009 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:56,036 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/B/7d2e9e37dc504eeb9565c8ad6bb9ef25 2024-12-07T15:13:56,047 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/C/369e0a4b3ede47369ec5f0c65257637f is 50, key is test_row_0/C:col10/1733584435551/Put/seqid=0 2024-12-07T15:13:56,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741905_1081 (size=12301) 2024-12-07T15:13:56,052 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/C/369e0a4b3ede47369ec5f0c65257637f 2024-12-07T15:13:56,060 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/A/4a12ef7335bc44a1873dd4f8b79e2e22 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/4a12ef7335bc44a1873dd4f8b79e2e22 2024-12-07T15:13:56,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-07T15:13:56,064 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T15:13:56,064 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T15:13:56,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-12-07T15:13:56,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-07T15:13:56,066 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/4a12ef7335bc44a1873dd4f8b79e2e22, entries=250, sequenceid=317, filesize=16.8 K 2024-12-07T15:13:56,067 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T15:13:56,067 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/B/7d2e9e37dc504eeb9565c8ad6bb9ef25 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/7d2e9e37dc504eeb9565c8ad6bb9ef25 2024-12-07T15:13:56,068 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T15:13:56,068 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T15:13:56,075 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/7d2e9e37dc504eeb9565c8ad6bb9ef25, entries=150, sequenceid=317, filesize=12.0 K 2024-12-07T15:13:56,077 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/C/369e0a4b3ede47369ec5f0c65257637f as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/369e0a4b3ede47369ec5f0c65257637f 2024-12-07T15:13:56,082 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/369e0a4b3ede47369ec5f0c65257637f, entries=150, sequenceid=317, filesize=12.0 K 2024-12-07T15:13:56,083 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 53401f0681ce510d042c1599ea9bc7e2 in 527ms, sequenceid=317, compaction requested=true 2024-12-07T15:13:56,083 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 53401f0681ce510d042c1599ea9bc7e2: 2024-12-07T15:13:56,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 53401f0681ce510d042c1599ea9bc7e2:A, priority=-2147483648, current under compaction store size is 1 2024-12-07T15:13:56,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:13:56,083 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:13:56,083 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:13:56,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 53401f0681ce510d042c1599ea9bc7e2:B, priority=-2147483648, current under compaction store size is 2 2024-12-07T15:13:56,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:13:56,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 53401f0681ce510d042c1599ea9bc7e2:C, priority=-2147483648, current under compaction store size is 3 2024-12-07T15:13:56,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T15:13:56,084 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42465 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:13:56,084 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:13:56,084 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1541): 53401f0681ce510d042c1599ea9bc7e2/A is initiating minor compaction (all files) 2024-12-07T15:13:56,085 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1541): 53401f0681ce510d042c1599ea9bc7e2/B is initiating minor compaction (all files) 2024-12-07T15:13:56,085 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 53401f0681ce510d042c1599ea9bc7e2/A in TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:56,085 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 53401f0681ce510d042c1599ea9bc7e2/B in TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:56,085 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/9dc7ea62c09f426d931bc23d362be474, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/eacf947c67264a109c5ccceb000c3bca, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/4a12ef7335bc44a1873dd4f8b79e2e22] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp, totalSize=41.5 K 2024-12-07T15:13:56,085 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/dd640f8ad6d9483cad6b4bdad1827b9b, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/2795874958f646d69ab7fe486e2e969c, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/7d2e9e37dc504eeb9565c8ad6bb9ef25] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp, totalSize=36.7 K 2024-12-07T15:13:56,085 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9dc7ea62c09f426d931bc23d362be474, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1733584432969 2024-12-07T15:13:56,086 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting dd640f8ad6d9483cad6b4bdad1827b9b, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1733584432969 2024-12-07T15:13:56,086 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting eacf947c67264a109c5ccceb000c3bca, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1733584433363 2024-12-07T15:13:56,086 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 2795874958f646d69ab7fe486e2e969c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1733584433363 2024-12-07T15:13:56,087 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a12ef7335bc44a1873dd4f8b79e2e22, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1733584435544 2024-12-07T15:13:56,087 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d2e9e37dc504eeb9565c8ad6bb9ef25, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1733584435544 2024-12-07T15:13:56,099 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 53401f0681ce510d042c1599ea9bc7e2#B#compaction#69 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:13:56,099 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/B/e479e6482dda4be8ad710c95097a11e2 is 50, key is test_row_0/B:col10/1733584435551/Put/seqid=0 2024-12-07T15:13:56,103 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 53401f0681ce510d042c1599ea9bc7e2#A#compaction#70 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:13:56,103 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/A/f7990215a0ca4783b0d9eac1aa73279e is 50, key is test_row_0/A:col10/1733584435551/Put/seqid=0 2024-12-07T15:13:56,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741906_1082 (size=13085) 2024-12-07T15:13:56,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741907_1083 (size=13085) 2024-12-07T15:13:56,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-07T15:13:56,221 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-07T15:13:56,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:56,221 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 53401f0681ce510d042c1599ea9bc7e2 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-07T15:13:56,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 53401f0681ce510d042c1599ea9bc7e2, store=A 2024-12-07T15:13:56,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T15:13:56,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 53401f0681ce510d042c1599ea9bc7e2, store=B 2024-12-07T15:13:56,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T15:13:56,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 53401f0681ce510d042c1599ea9bc7e2, store=C 2024-12-07T15:13:56,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T15:13:56,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/A/58a57450de9b4228b69085609e8a811f is 50, key is test_row_0/A:col10/1733584435676/Put/seqid=0 2024-12-07T15:13:56,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741908_1084 (size=12301) 2024-12-07T15:13:56,233 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/A/58a57450de9b4228b69085609e8a811f 2024-12-07T15:13:56,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/B/d3687378c6044ca9a27c3aa901642194 is 50, key is test_row_0/B:col10/1733584435676/Put/seqid=0 2024-12-07T15:13:56,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741909_1085 (size=12301) 2024-12-07T15:13:56,247 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/B/d3687378c6044ca9a27c3aa901642194 2024-12-07T15:13:56,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/C/7e1c3628292e485fbf9a3fe35a198bfe is 50, key is test_row_0/C:col10/1733584435676/Put/seqid=0 2024-12-07T15:13:56,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741910_1086 (size=12301) 2024-12-07T15:13:56,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on 53401f0681ce510d042c1599ea9bc7e2 2024-12-07T15:13:56,305 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. as already flushing 2024-12-07T15:13:56,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:56,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1733584496326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:56,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:56,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:56,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:56,330 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:56,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36040 deadline: 1733584496327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:56,331 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:56,331 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:56,331 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:56,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-07T15:13:56,433 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:56,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1733584496433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:56,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:56,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:56,434 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:56,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:56,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36040 deadline: 1733584496433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:56,435 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:56,435 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:56,435 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:56,521 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/B/e479e6482dda4be8ad710c95097a11e2 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/e479e6482dda4be8ad710c95097a11e2 2024-12-07T15:13:56,522 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/A/f7990215a0ca4783b0d9eac1aa73279e as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/f7990215a0ca4783b0d9eac1aa73279e 2024-12-07T15:13:56,530 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 53401f0681ce510d042c1599ea9bc7e2/A of 53401f0681ce510d042c1599ea9bc7e2 into f7990215a0ca4783b0d9eac1aa73279e(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:13:56,530 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 53401f0681ce510d042c1599ea9bc7e2/B of 53401f0681ce510d042c1599ea9bc7e2 into e479e6482dda4be8ad710c95097a11e2(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:13:56,530 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 53401f0681ce510d042c1599ea9bc7e2: 2024-12-07T15:13:56,530 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 53401f0681ce510d042c1599ea9bc7e2: 2024-12-07T15:13:56,531 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., storeName=53401f0681ce510d042c1599ea9bc7e2/A, priority=13, startTime=1733584436083; duration=0sec 2024-12-07T15:13:56,531 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., storeName=53401f0681ce510d042c1599ea9bc7e2/B, priority=13, startTime=1733584436083; duration=0sec 2024-12-07T15:13:56,531 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T15:13:56,531 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 53401f0681ce510d042c1599ea9bc7e2:A 2024-12-07T15:13:56,531 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:13:56,531 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:13:56,531 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 53401f0681ce510d042c1599ea9bc7e2:B 2024-12-07T15:13:56,532 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:13:56,533 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1541): 53401f0681ce510d042c1599ea9bc7e2/C is initiating minor compaction (all files) 2024-12-07T15:13:56,533 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 53401f0681ce510d042c1599ea9bc7e2/C in TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:56,533 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/87d27ff4e6764394b111eee75e3daf8a, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/c0f918b435074fe3b366dd0923e81539, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/369e0a4b3ede47369ec5f0c65257637f] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp, totalSize=36.7 K 2024-12-07T15:13:56,533 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 87d27ff4e6764394b111eee75e3daf8a, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1733584432969 2024-12-07T15:13:56,534 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting c0f918b435074fe3b366dd0923e81539, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1733584433363 2024-12-07T15:13:56,534 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 369e0a4b3ede47369ec5f0c65257637f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1733584435544 2024-12-07T15:13:56,542 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 53401f0681ce510d042c1599ea9bc7e2#C#compaction#74 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:13:56,543 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/C/cf8a019e31b948adbd5eb210b0d8f975 is 50, key is test_row_0/C:col10/1733584435551/Put/seqid=0 2024-12-07T15:13:56,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741911_1087 (size=13085) 2024-12-07T15:13:56,649 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:56,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1733584496643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:56,649 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:56,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:56,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36040 deadline: 1733584496643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:56,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:56,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:56,650 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:56,650 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:56,650 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:56,662 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/C/7e1c3628292e485fbf9a3fe35a198bfe 2024-12-07T15:13:56,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/A/58a57450de9b4228b69085609e8a811f as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/58a57450de9b4228b69085609e8a811f 2024-12-07T15:13:56,677 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/58a57450de9b4228b69085609e8a811f, entries=150, sequenceid=354, filesize=12.0 K 2024-12-07T15:13:56,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/B/d3687378c6044ca9a27c3aa901642194 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/d3687378c6044ca9a27c3aa901642194 2024-12-07T15:13:56,683 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/d3687378c6044ca9a27c3aa901642194, entries=150, sequenceid=354, filesize=12.0 K 2024-12-07T15:13:56,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/C/7e1c3628292e485fbf9a3fe35a198bfe as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/7e1c3628292e485fbf9a3fe35a198bfe 2024-12-07T15:13:56,691 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/7e1c3628292e485fbf9a3fe35a198bfe, entries=150, sequenceid=354, filesize=12.0 K 2024-12-07T15:13:56,692 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 53401f0681ce510d042c1599ea9bc7e2 in 471ms, sequenceid=354, compaction requested=false 2024-12-07T15:13:56,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 53401f0681ce510d042c1599ea9bc7e2: 2024-12-07T15:13:56,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:56,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-12-07T15:13:56,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-07T15:13:56,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-12-07T15:13:56,697 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-12-07T15:13:56,697 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 626 msec 2024-12-07T15:13:56,699 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 633 msec 2024-12-07T15:13:56,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on 53401f0681ce510d042c1599ea9bc7e2 2024-12-07T15:13:56,956 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 53401f0681ce510d042c1599ea9bc7e2 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-07T15:13:56,956 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 53401f0681ce510d042c1599ea9bc7e2, store=A 2024-12-07T15:13:56,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:13:56,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 53401f0681ce510d042c1599ea9bc7e2, store=B 2024-12-07T15:13:56,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:13:56,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 53401f0681ce510d042c1599ea9bc7e2, store=C 2024-12-07T15:13:56,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:13:56,959 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/C/cf8a019e31b948adbd5eb210b0d8f975 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/cf8a019e31b948adbd5eb210b0d8f975 2024-12-07T15:13:56,963 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/A/4ab84ea3fe914664be05058468299eed is 50, key is test_row_0/A:col10/1733584436304/Put/seqid=0 2024-12-07T15:13:56,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741912_1088 (size=14741) 2024-12-07T15:13:56,971 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 53401f0681ce510d042c1599ea9bc7e2/C of 53401f0681ce510d042c1599ea9bc7e2 into cf8a019e31b948adbd5eb210b0d8f975(size=12.8 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:13:56,971 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 53401f0681ce510d042c1599ea9bc7e2: 2024-12-07T15:13:56,971 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., storeName=53401f0681ce510d042c1599ea9bc7e2/C, priority=13, startTime=1733584436083; duration=0sec 2024-12-07T15:13:56,971 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:13:56,971 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 53401f0681ce510d042c1599ea9bc7e2:C 2024-12-07T15:13:57,040 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733584436985, max timestamp=1733584437034Num uniques 859; 2024-12-07T15:13:57,040 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733584436985, max timestamp=1733584437034Num uniques 859; 2024-12-07T15:13:57,040 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-07T15:13:57,040 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-07T15:13:57,042 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733584436985, max timestamp=1733584437034Num uniques 859; 2024-12-07T15:13:57,043 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-07T15:13:57,091 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:57,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36040 deadline: 1733584497086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:57,092 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:57,092 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:57,092 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:57,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:57,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1733584497091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:57,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:57,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:57,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:57,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-07T15:13:57,206 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T15:13:57,206 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T15:13:57,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-12-07T15:13:57,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-07T15:13:57,210 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T15:13:57,210 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:57,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36040 deadline: 1733584497205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:57,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:57,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1733584497205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:57,212 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T15:13:57,212 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T15:13:57,213 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:57,213 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:57,213 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:57,213 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:57,213 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:57,213 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:57,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-07T15:13:57,365 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-07T15:13:57,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:57,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. as already flushing 2024-12-07T15:13:57,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:57,366 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:13:57,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:13:57,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:13:57,371 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/A/4ab84ea3fe914664be05058468299eed 2024-12-07T15:13:57,380 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/B/6cd47374edb144478cfef41ba51bf1a1 is 50, key is test_row_0/B:col10/1733584436304/Put/seqid=0 2024-12-07T15:13:57,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741913_1089 (size=12301) 2024-12-07T15:13:57,387 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/B/6cd47374edb144478cfef41ba51bf1a1 2024-12-07T15:13:57,395 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/C/1f5eaa5ea0b2475993f52c0070bb06a9 is 50, key is test_row_0/C:col10/1733584436304/Put/seqid=0 2024-12-07T15:13:57,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741914_1090 (size=12301) 2024-12-07T15:13:57,400 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/C/1f5eaa5ea0b2475993f52c0070bb06a9 2024-12-07T15:13:57,407 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/A/4ab84ea3fe914664be05058468299eed as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/4ab84ea3fe914664be05058468299eed 2024-12-07T15:13:57,413 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/4ab84ea3fe914664be05058468299eed, entries=200, sequenceid=367, filesize=14.4 K 2024-12-07T15:13:57,415 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/B/6cd47374edb144478cfef41ba51bf1a1 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/6cd47374edb144478cfef41ba51bf1a1 2024-12-07T15:13:57,420 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/6cd47374edb144478cfef41ba51bf1a1, entries=150, sequenceid=367, filesize=12.0 K 2024-12-07T15:13:57,421 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/C/1f5eaa5ea0b2475993f52c0070bb06a9 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/1f5eaa5ea0b2475993f52c0070bb06a9 2024-12-07T15:13:57,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:57,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36040 deadline: 1733584497424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:57,426 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:57,426 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:57,426 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:57,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:57,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1733584497424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:57,428 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/1f5eaa5ea0b2475993f52c0070bb06a9, entries=150, sequenceid=367, filesize=12.0 K 2024-12-07T15:13:57,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:57,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:57,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:57,429 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 53401f0681ce510d042c1599ea9bc7e2 in 473ms, sequenceid=367, compaction requested=true 2024-12-07T15:13:57,429 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 53401f0681ce510d042c1599ea9bc7e2: 2024-12-07T15:13:57,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 53401f0681ce510d042c1599ea9bc7e2:A, priority=-2147483648, current under compaction store size is 1 2024-12-07T15:13:57,430 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:13:57,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:13:57,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 53401f0681ce510d042c1599ea9bc7e2:B, priority=-2147483648, current under compaction store size is 2 2024-12-07T15:13:57,430 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:13:57,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:13:57,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 53401f0681ce510d042c1599ea9bc7e2:C, priority=-2147483648, current under compaction store size is 3 2024-12-07T15:13:57,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T15:13:57,433 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40127 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:13:57,433 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1541): 53401f0681ce510d042c1599ea9bc7e2/A is initiating minor compaction (all files) 2024-12-07T15:13:57,433 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 53401f0681ce510d042c1599ea9bc7e2/A in TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:57,433 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/f7990215a0ca4783b0d9eac1aa73279e, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/58a57450de9b4228b69085609e8a811f, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/4ab84ea3fe914664be05058468299eed] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp, totalSize=39.2 K 2024-12-07T15:13:57,434 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:13:57,434 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting f7990215a0ca4783b0d9eac1aa73279e, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1733584435544 2024-12-07T15:13:57,434 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1541): 53401f0681ce510d042c1599ea9bc7e2/B is initiating minor compaction (all files) 2024-12-07T15:13:57,434 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 53401f0681ce510d042c1599ea9bc7e2/B in TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:57,434 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/e479e6482dda4be8ad710c95097a11e2, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/d3687378c6044ca9a27c3aa901642194, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/6cd47374edb144478cfef41ba51bf1a1] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp, totalSize=36.8 K 2024-12-07T15:13:57,434 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 58a57450de9b4228b69085609e8a811f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1733584435635 2024-12-07T15:13:57,435 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ab84ea3fe914664be05058468299eed, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1733584436304 2024-12-07T15:13:57,435 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting e479e6482dda4be8ad710c95097a11e2, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1733584435544 2024-12-07T15:13:57,435 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting d3687378c6044ca9a27c3aa901642194, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1733584435635 2024-12-07T15:13:57,436 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 6cd47374edb144478cfef41ba51bf1a1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1733584436304 2024-12-07T15:13:57,445 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 53401f0681ce510d042c1599ea9bc7e2#A#compaction#78 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:13:57,446 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/A/05da278fb6274982a29989f83cf28a6a is 50, key is test_row_0/A:col10/1733584436304/Put/seqid=0 2024-12-07T15:13:57,449 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 53401f0681ce510d042c1599ea9bc7e2#B#compaction#79 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:13:57,449 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/B/21d198e4d0f44c5faa703daecf3e80ef is 50, key is test_row_0/B:col10/1733584436304/Put/seqid=0 2024-12-07T15:13:57,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741915_1091 (size=13187) 2024-12-07T15:13:57,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741916_1092 (size=13187) 2024-12-07T15:13:57,520 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-07T15:13:57,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:57,521 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing 53401f0681ce510d042c1599ea9bc7e2 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-12-07T15:13:57,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 53401f0681ce510d042c1599ea9bc7e2, store=A 2024-12-07T15:13:57,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T15:13:57,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 53401f0681ce510d042c1599ea9bc7e2, store=B 2024-12-07T15:13:57,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T15:13:57,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 53401f0681ce510d042c1599ea9bc7e2, store=C 2024-12-07T15:13:57,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T15:13:57,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-07T15:13:57,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/A/f401219dffb941d798f02420c82d5be7 is 50, key is test_row_0/A:col10/1733584437074/Put/seqid=0 2024-12-07T15:13:57,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741917_1093 (size=12301) 2024-12-07T15:13:57,533 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/A/f401219dffb941d798f02420c82d5be7 2024-12-07T15:13:57,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/B/9341a047f50945deb2f8b52293339f54 is 50, key is test_row_0/B:col10/1733584437074/Put/seqid=0 2024-12-07T15:13:57,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741918_1094 (size=12301) 2024-12-07T15:13:57,553 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/B/9341a047f50945deb2f8b52293339f54 2024-12-07T15:13:57,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/C/ab8a7cde8a6c457496f39f137cec83cd is 50, key is test_row_0/C:col10/1733584437074/Put/seqid=0 2024-12-07T15:13:57,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741919_1095 (size=12301) 2024-12-07T15:13:57,578 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/C/ab8a7cde8a6c457496f39f137cec83cd 2024-12-07T15:13:57,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/A/f401219dffb941d798f02420c82d5be7 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/f401219dffb941d798f02420c82d5be7 2024-12-07T15:13:57,593 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/f401219dffb941d798f02420c82d5be7, entries=150, sequenceid=404, filesize=12.0 K 2024-12-07T15:13:57,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/B/9341a047f50945deb2f8b52293339f54 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/9341a047f50945deb2f8b52293339f54 2024-12-07T15:13:57,605 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/9341a047f50945deb2f8b52293339f54, entries=150, sequenceid=404, filesize=12.0 K 2024-12-07T15:13:57,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/C/ab8a7cde8a6c457496f39f137cec83cd as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/ab8a7cde8a6c457496f39f137cec83cd 2024-12-07T15:13:57,613 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/ab8a7cde8a6c457496f39f137cec83cd, entries=150, sequenceid=404, filesize=12.0 K 2024-12-07T15:13:57,614 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=0 B/0 for 53401f0681ce510d042c1599ea9bc7e2 in 94ms, sequenceid=404, compaction requested=true 2024-12-07T15:13:57,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for 53401f0681ce510d042c1599ea9bc7e2: 2024-12-07T15:13:57,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:57,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-12-07T15:13:57,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-12-07T15:13:57,618 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-12-07T15:13:57,619 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 404 msec 2024-12-07T15:13:57,621 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 414 msec 2024-12-07T15:13:57,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on 53401f0681ce510d042c1599ea9bc7e2 2024-12-07T15:13:57,746 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 53401f0681ce510d042c1599ea9bc7e2 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-07T15:13:57,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 53401f0681ce510d042c1599ea9bc7e2, store=A 2024-12-07T15:13:57,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:13:57,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 53401f0681ce510d042c1599ea9bc7e2, store=B 2024-12-07T15:13:57,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:13:57,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 53401f0681ce510d042c1599ea9bc7e2, store=C 2024-12-07T15:13:57,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:13:57,752 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/A/f37911ada0f04e0f9514fee229ea1d14 is 50, key is test_row_0/A:col10/1733584437745/Put/seqid=0 2024-12-07T15:13:57,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741920_1096 (size=22065) 2024-12-07T15:13:57,815 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733584437770, max timestamp=1733584437811Num uniques 859; 2024-12-07T15:13:57,815 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733584437770, max timestamp=1733584437811Num uniques 859; 2024-12-07T15:13:57,815 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733584437770, max timestamp=1733584437811Num uniques 859; 2024-12-07T15:13:57,815 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-07T15:13:57,815 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-07T15:13:57,815 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-07T15:13:57,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-07T15:13:57,834 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T15:13:57,835 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T15:13:57,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-12-07T15:13:57,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-07T15:13:57,839 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T15:13:57,840 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T15:13:57,840 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T15:13:57,863 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/A/05da278fb6274982a29989f83cf28a6a as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/05da278fb6274982a29989f83cf28a6a 2024-12-07T15:13:57,870 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/B/21d198e4d0f44c5faa703daecf3e80ef as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/21d198e4d0f44c5faa703daecf3e80ef 2024-12-07T15:13:57,872 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 53401f0681ce510d042c1599ea9bc7e2/A of 53401f0681ce510d042c1599ea9bc7e2 into 05da278fb6274982a29989f83cf28a6a(size=12.9 K), total size for store is 24.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:13:57,872 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 53401f0681ce510d042c1599ea9bc7e2: 2024-12-07T15:13:57,872 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., storeName=53401f0681ce510d042c1599ea9bc7e2/A, priority=13, startTime=1733584437429; duration=0sec 2024-12-07T15:13:57,873 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T15:13:57,873 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 53401f0681ce510d042c1599ea9bc7e2:A 2024-12-07T15:13:57,873 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-07T15:13:57,876 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-07T15:13:57,876 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1541): 53401f0681ce510d042c1599ea9bc7e2/C is initiating minor compaction (all files) 2024-12-07T15:13:57,877 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 53401f0681ce510d042c1599ea9bc7e2/C in TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:57,877 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/cf8a019e31b948adbd5eb210b0d8f975, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/7e1c3628292e485fbf9a3fe35a198bfe, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/1f5eaa5ea0b2475993f52c0070bb06a9, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/ab8a7cde8a6c457496f39f137cec83cd] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp, totalSize=48.8 K 2024-12-07T15:13:57,877 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:57,877 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf8a019e31b948adbd5eb210b0d8f975, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1733584435544 2024-12-07T15:13:57,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36040 deadline: 1733584497864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:57,878 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:57,878 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e1c3628292e485fbf9a3fe35a198bfe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1733584435635 2024-12-07T15:13:57,878 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:57,878 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:57,878 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 53401f0681ce510d042c1599ea9bc7e2/B of 53401f0681ce510d042c1599ea9bc7e2 into 21d198e4d0f44c5faa703daecf3e80ef(size=12.9 K), total size for store is 24.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:13:57,878 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 53401f0681ce510d042c1599ea9bc7e2: 2024-12-07T15:13:57,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:57,879 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f5eaa5ea0b2475993f52c0070bb06a9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1733584436304 2024-12-07T15:13:57,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1733584497867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:57,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:57,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:57,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:57,879 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab8a7cde8a6c457496f39f137cec83cd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1733584437074 2024-12-07T15:13:57,880 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., storeName=53401f0681ce510d042c1599ea9bc7e2/B, priority=13, startTime=1733584437430; duration=0sec 2024-12-07T15:13:57,881 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:13:57,882 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 53401f0681ce510d042c1599ea9bc7e2:B 2024-12-07T15:13:57,897 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 53401f0681ce510d042c1599ea9bc7e2#C#compaction#84 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:13:57,898 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/C/4262fec1d81f4ee9846791a37a2402cf is 50, key is test_row_0/C:col10/1733584437074/Put/seqid=0 2024-12-07T15:13:57,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741921_1097 (size=13221) 2024-12-07T15:13:57,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-07T15:13:57,988 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:57,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36040 deadline: 1733584497983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:57,989 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:57,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:57,989 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:57,989 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:57,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1733584497983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:57,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:57,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:57,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:57,993 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-07T15:13:57,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:57,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. as already flushing 2024-12-07T15:13:57,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:57,994 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:13:57,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:13:57,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:13:58,147 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-07T15:13:58,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:58,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. as already flushing 2024-12-07T15:13:58,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:58,147 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:13:58,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:13:58,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:13:58,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-07T15:13:58,158 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/A/f37911ada0f04e0f9514fee229ea1d14 2024-12-07T15:13:58,169 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/B/ad1fa4ed689e493e9e3b0442fb909386 is 50, key is test_row_0/B:col10/1733584437745/Put/seqid=0 2024-12-07T15:13:58,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741922_1098 (size=12301) 2024-12-07T15:13:58,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:58,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36040 deadline: 1733584498193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:58,203 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:58,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:58,203 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:58,203 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:58,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1733584498193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:58,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:58,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:58,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:58,300 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-07T15:13:58,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:58,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. as already flushing 2024-12-07T15:13:58,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:58,301 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:13:58,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:13:58,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:13:58,318 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/C/4262fec1d81f4ee9846791a37a2402cf as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/4262fec1d81f4ee9846791a37a2402cf 2024-12-07T15:13:58,326 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 53401f0681ce510d042c1599ea9bc7e2/C of 53401f0681ce510d042c1599ea9bc7e2 into 4262fec1d81f4ee9846791a37a2402cf(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:13:58,326 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 53401f0681ce510d042c1599ea9bc7e2: 2024-12-07T15:13:58,326 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., storeName=53401f0681ce510d042c1599ea9bc7e2/C, priority=12, startTime=1733584437430; duration=0sec 2024-12-07T15:13:58,326 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:13:58,326 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 53401f0681ce510d042c1599ea9bc7e2:C 2024-12-07T15:13:58,455 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-07T15:13:58,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:58,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. as already flushing 2024-12-07T15:13:58,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:58,455 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:13:58,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:13:58,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:13:58,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-07T15:13:58,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:58,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36040 deadline: 1733584498513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:58,516 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:58,516 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:58,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:58,516 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:58,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1733584498513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:58,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:58,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:58,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:58,575 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/B/ad1fa4ed689e493e9e3b0442fb909386 2024-12-07T15:13:58,583 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/C/2fa0e85e039f40f69e0128098049e403 is 50, key is test_row_0/C:col10/1733584437745/Put/seqid=0 2024-12-07T15:13:58,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741923_1099 (size=12301) 2024-12-07T15:13:58,591 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/C/2fa0e85e039f40f69e0128098049e403 2024-12-07T15:13:58,598 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/A/f37911ada0f04e0f9514fee229ea1d14 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/f37911ada0f04e0f9514fee229ea1d14 2024-12-07T15:13:58,603 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/f37911ada0f04e0f9514fee229ea1d14, entries=350, sequenceid=415, filesize=21.5 K 2024-12-07T15:13:58,604 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/B/ad1fa4ed689e493e9e3b0442fb909386 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/ad1fa4ed689e493e9e3b0442fb909386 2024-12-07T15:13:58,610 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/ad1fa4ed689e493e9e3b0442fb909386, entries=150, sequenceid=415, filesize=12.0 K 2024-12-07T15:13:58,611 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-07T15:13:58,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:58,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. as already flushing 2024-12-07T15:13:58,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:58,611 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:13:58,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:13:58,612 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/C/2fa0e85e039f40f69e0128098049e403 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/2fa0e85e039f40f69e0128098049e403 2024-12-07T15:13:58,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:13:58,619 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/2fa0e85e039f40f69e0128098049e403, entries=150, sequenceid=415, filesize=12.0 K 2024-12-07T15:13:58,621 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 53401f0681ce510d042c1599ea9bc7e2 in 876ms, sequenceid=415, compaction requested=true 2024-12-07T15:13:58,621 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 53401f0681ce510d042c1599ea9bc7e2: 2024-12-07T15:13:58,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 53401f0681ce510d042c1599ea9bc7e2:A, priority=-2147483648, current under compaction store size is 1 2024-12-07T15:13:58,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:13:58,621 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:13:58,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 53401f0681ce510d042c1599ea9bc7e2:B, priority=-2147483648, current under compaction store size is 2 2024-12-07T15:13:58,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:13:58,622 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:13:58,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 53401f0681ce510d042c1599ea9bc7e2:C, priority=-2147483648, current under compaction store size is 3 2024-12-07T15:13:58,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T15:13:58,623 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 47553 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:13:58,623 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:13:58,623 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1541): 53401f0681ce510d042c1599ea9bc7e2/A is initiating minor compaction (all files) 2024-12-07T15:13:58,623 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1541): 53401f0681ce510d042c1599ea9bc7e2/B is initiating minor compaction (all files) 2024-12-07T15:13:58,623 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 53401f0681ce510d042c1599ea9bc7e2/A in TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:58,623 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 53401f0681ce510d042c1599ea9bc7e2/B in TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:58,623 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/05da278fb6274982a29989f83cf28a6a, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/f401219dffb941d798f02420c82d5be7, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/f37911ada0f04e0f9514fee229ea1d14] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp, totalSize=46.4 K 2024-12-07T15:13:58,623 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/21d198e4d0f44c5faa703daecf3e80ef, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/9341a047f50945deb2f8b52293339f54, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/ad1fa4ed689e493e9e3b0442fb909386] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp, totalSize=36.9 K 2024-12-07T15:13:58,624 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05da278fb6274982a29989f83cf28a6a, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1733584436304 2024-12-07T15:13:58,624 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting f401219dffb941d798f02420c82d5be7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1733584437074 2024-12-07T15:13:58,624 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 21d198e4d0f44c5faa703daecf3e80ef, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1733584436304 2024-12-07T15:13:58,625 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting f37911ada0f04e0f9514fee229ea1d14, keycount=350, bloomtype=ROW, size=21.5 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1733584437734 2024-12-07T15:13:58,625 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 9341a047f50945deb2f8b52293339f54, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1733584437074 2024-12-07T15:13:58,626 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting ad1fa4ed689e493e9e3b0442fb909386, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1733584437737 2024-12-07T15:13:58,635 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 53401f0681ce510d042c1599ea9bc7e2#B#compaction#87 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:13:58,636 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/B/8fc90821f61c4378a27b717e06c4a337 is 50, key is test_row_0/B:col10/1733584437745/Put/seqid=0 2024-12-07T15:13:58,642 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 53401f0681ce510d042c1599ea9bc7e2#A#compaction#88 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:13:58,643 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/A/bf77e1f312814f06b3fb99b44425174e is 50, key is test_row_0/A:col10/1733584437745/Put/seqid=0 2024-12-07T15:13:58,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741924_1100 (size=13289) 2024-12-07T15:13:58,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741925_1101 (size=13289) 2024-12-07T15:13:58,765 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-07T15:13:58,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:13:58,765 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing 53401f0681ce510d042c1599ea9bc7e2 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-12-07T15:13:58,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 53401f0681ce510d042c1599ea9bc7e2, store=A 2024-12-07T15:13:58,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T15:13:58,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 53401f0681ce510d042c1599ea9bc7e2, store=B 2024-12-07T15:13:58,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T15:13:58,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 53401f0681ce510d042c1599ea9bc7e2, store=C 2024-12-07T15:13:58,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T15:13:58,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/A/67162cecbba8410c99abf042d4869aa0 is 50, key is test_row_0/A:col10/1733584437860/Put/seqid=0 2024-12-07T15:13:58,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741926_1102 (size=12301) 2024-12-07T15:13:58,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-07T15:13:59,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on 53401f0681ce510d042c1599ea9bc7e2 2024-12-07T15:13:59,027 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. as already flushing 2024-12-07T15:13:59,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:59,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 221 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36040 deadline: 1733584499044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:59,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:59,048 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:59,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1733584499045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:59,048 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:59,048 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:59,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:59,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:59,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:59,055 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/B/8fc90821f61c4378a27b717e06c4a337 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/8fc90821f61c4378a27b717e06c4a337 2024-12-07T15:13:59,055 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/A/bf77e1f312814f06b3fb99b44425174e as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/bf77e1f312814f06b3fb99b44425174e 2024-12-07T15:13:59,064 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 53401f0681ce510d042c1599ea9bc7e2/B of 53401f0681ce510d042c1599ea9bc7e2 into 8fc90821f61c4378a27b717e06c4a337(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:13:59,064 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 53401f0681ce510d042c1599ea9bc7e2/A of 53401f0681ce510d042c1599ea9bc7e2 into bf77e1f312814f06b3fb99b44425174e(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:13:59,064 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 53401f0681ce510d042c1599ea9bc7e2: 2024-12-07T15:13:59,064 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 53401f0681ce510d042c1599ea9bc7e2: 2024-12-07T15:13:59,064 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., storeName=53401f0681ce510d042c1599ea9bc7e2/A, priority=13, startTime=1733584438621; duration=0sec 2024-12-07T15:13:59,064 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., storeName=53401f0681ce510d042c1599ea9bc7e2/B, priority=13, startTime=1733584438621; duration=0sec 2024-12-07T15:13:59,064 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T15:13:59,064 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 53401f0681ce510d042c1599ea9bc7e2:B 2024-12-07T15:13:59,064 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T15:13:59,064 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 53401f0681ce510d042c1599ea9bc7e2:A 2024-12-07T15:13:59,064 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-07T15:13:59,065 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-07T15:13:59,065 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-07T15:13:59,066 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. because compaction request was cancelled 2024-12-07T15:13:59,066 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 53401f0681ce510d042c1599ea9bc7e2:C 2024-12-07T15:13:59,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:59,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36040 deadline: 1733584499153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:59,157 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:59,157 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:59,157 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:59,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:59,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1733584499153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:59,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:59,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:59,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:59,176 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=452 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/A/67162cecbba8410c99abf042d4869aa0 2024-12-07T15:13:59,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/B/e05e1992037a4b8a87b707918eb3f4f3 is 50, key is test_row_0/B:col10/1733584437860/Put/seqid=0 2024-12-07T15:13:59,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741927_1103 (size=12301) 2024-12-07T15:13:59,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:59,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36040 deadline: 1733584499363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:59,370 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:59,370 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:59,370 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:59,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:59,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1733584499373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:59,375 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:59,375 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:59,375 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:59,591 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=452 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/B/e05e1992037a4b8a87b707918eb3f4f3 2024-12-07T15:13:59,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/C/e6cbb92d0359454da666ec4ce16bf59b is 50, key is test_row_0/C:col10/1733584437860/Put/seqid=0 2024-12-07T15:13:59,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741928_1104 (size=12301) 2024-12-07T15:13:59,685 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:59,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36040 deadline: 1733584499683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:59,686 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:59,686 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:59,686 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:59,686 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:13:59,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1733584499684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:13:59,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:59,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:13:59,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:13:59,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-07T15:14:00,005 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=452 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/C/e6cbb92d0359454da666ec4ce16bf59b 2024-12-07T15:14:00,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/A/67162cecbba8410c99abf042d4869aa0 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/67162cecbba8410c99abf042d4869aa0 2024-12-07T15:14:00,016 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/67162cecbba8410c99abf042d4869aa0, entries=150, sequenceid=452, filesize=12.0 K 2024-12-07T15:14:00,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/B/e05e1992037a4b8a87b707918eb3f4f3 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/e05e1992037a4b8a87b707918eb3f4f3 2024-12-07T15:14:00,022 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/e05e1992037a4b8a87b707918eb3f4f3, entries=150, sequenceid=452, filesize=12.0 K 2024-12-07T15:14:00,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/C/e6cbb92d0359454da666ec4ce16bf59b as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/e6cbb92d0359454da666ec4ce16bf59b 2024-12-07T15:14:00,028 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/e6cbb92d0359454da666ec4ce16bf59b, entries=150, sequenceid=452, filesize=12.0 K 2024-12-07T15:14:00,029 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~355.64 KB/364176, currentSize=53.67 KB/54960 for 53401f0681ce510d042c1599ea9bc7e2 in 1264ms, sequenceid=452, compaction requested=true 2024-12-07T15:14:00,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for 53401f0681ce510d042c1599ea9bc7e2: 2024-12-07T15:14:00,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:14:00,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-12-07T15:14:00,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-12-07T15:14:00,033 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-12-07T15:14:00,033 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1910 sec 2024-12-07T15:14:00,035 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 2.1990 sec 2024-12-07T15:14:00,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on 53401f0681ce510d042c1599ea9bc7e2 2024-12-07T15:14:00,198 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 53401f0681ce510d042c1599ea9bc7e2 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-07T15:14:00,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 53401f0681ce510d042c1599ea9bc7e2, store=A 2024-12-07T15:14:00,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:14:00,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 53401f0681ce510d042c1599ea9bc7e2, store=B 2024-12-07T15:14:00,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:14:00,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 53401f0681ce510d042c1599ea9bc7e2, store=C 2024-12-07T15:14:00,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:14:00,203 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/A/bbff0d0c53ec4254bb8dc141577facb3 is 50, key is test_row_0/A:col10/1733584440196/Put/seqid=0 2024-12-07T15:14:00,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741929_1105 (size=17181) 2024-12-07T15:14:00,245 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:00,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 234 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36040 deadline: 1733584500242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:00,246 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:00,246 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:00,246 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:00,246 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:00,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 225 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1733584500245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:00,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:00,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:00,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:00,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:00,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 226 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1733584500353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:00,356 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:00,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:00,356 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:00,356 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:00,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 235 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36040 deadline: 1733584500354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:00,357 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:00,357 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:00,357 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:00,564 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:00,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1733584500564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:00,564 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:00,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:00,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:00,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 236 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36040 deadline: 1733584500564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:00,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:00,565 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:00,565 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:00,565 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:00,609 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=467 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/A/bbff0d0c53ec4254bb8dc141577facb3 2024-12-07T15:14:00,622 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/B/8104550a8c0941bcb193e3d73aa3abbe is 50, key is test_row_0/B:col10/1733584440196/Put/seqid=0 2024-12-07T15:14:00,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741930_1106 (size=12301) 2024-12-07T15:14:00,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:00,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 228 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1733584500873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:00,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:00,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:00,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 237 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36040 deadline: 1733584500873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:00,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:00,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:00,877 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:00,877 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:00,877 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:01,032 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=467 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/B/8104550a8c0941bcb193e3d73aa3abbe 2024-12-07T15:14:01,044 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/C/e96dacee72864a998b77a09410a09df1 is 50, key is test_row_0/C:col10/1733584440196/Put/seqid=0 2024-12-07T15:14:01,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741931_1107 (size=12301) 2024-12-07T15:14:01,386 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:01,386 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:01,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 229 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36058 deadline: 1733584501384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:01,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 238 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36040 deadline: 1733584501384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:01,387 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:01,387 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:01,387 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:01,388 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:01,388 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:01,388 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:01,452 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=467 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/C/e96dacee72864a998b77a09410a09df1 2024-12-07T15:14:01,460 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/A/bbff0d0c53ec4254bb8dc141577facb3 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/bbff0d0c53ec4254bb8dc141577facb3 2024-12-07T15:14:01,466 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/bbff0d0c53ec4254bb8dc141577facb3, entries=250, sequenceid=467, filesize=16.8 K 2024-12-07T15:14:01,467 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/B/8104550a8c0941bcb193e3d73aa3abbe as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/8104550a8c0941bcb193e3d73aa3abbe 2024-12-07T15:14:01,474 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/8104550a8c0941bcb193e3d73aa3abbe, entries=150, sequenceid=467, filesize=12.0 K 2024-12-07T15:14:01,476 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/C/e96dacee72864a998b77a09410a09df1 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/e96dacee72864a998b77a09410a09df1 2024-12-07T15:14:01,484 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/e96dacee72864a998b77a09410a09df1, entries=150, sequenceid=467, filesize=12.0 K 2024-12-07T15:14:01,485 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 53401f0681ce510d042c1599ea9bc7e2 in 1286ms, sequenceid=467, compaction requested=true 2024-12-07T15:14:01,485 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 53401f0681ce510d042c1599ea9bc7e2: 2024-12-07T15:14:01,485 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:14:01,485 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 53401f0681ce510d042c1599ea9bc7e2:A, priority=-2147483648, current under compaction store size is 1 2024-12-07T15:14:01,486 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:14:01,486 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:14:01,486 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 53401f0681ce510d042c1599ea9bc7e2:B, priority=-2147483648, current under compaction store size is 2 2024-12-07T15:14:01,486 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:14:01,486 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42771 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:14:01,486 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 53401f0681ce510d042c1599ea9bc7e2:C, priority=-2147483648, current under compaction store size is 3 2024-12-07T15:14:01,486 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T15:14:01,486 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1541): 53401f0681ce510d042c1599ea9bc7e2/A is initiating minor compaction (all files) 2024-12-07T15:14:01,486 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 53401f0681ce510d042c1599ea9bc7e2/A in TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:14:01,487 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/bf77e1f312814f06b3fb99b44425174e, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/67162cecbba8410c99abf042d4869aa0, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/bbff0d0c53ec4254bb8dc141577facb3] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp, totalSize=41.8 K 2024-12-07T15:14:01,487 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf77e1f312814f06b3fb99b44425174e, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1733584437737 2024-12-07T15:14:01,488 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:14:01,488 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1541): 53401f0681ce510d042c1599ea9bc7e2/B is initiating minor compaction (all files) 2024-12-07T15:14:01,488 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 53401f0681ce510d042c1599ea9bc7e2/B in TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:14:01,488 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/8fc90821f61c4378a27b717e06c4a337, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/e05e1992037a4b8a87b707918eb3f4f3, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/8104550a8c0941bcb193e3d73aa3abbe] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp, totalSize=37.0 K 2024-12-07T15:14:01,489 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 67162cecbba8410c99abf042d4869aa0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=452, earliestPutTs=1733584437855 2024-12-07T15:14:01,489 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 8fc90821f61c4378a27b717e06c4a337, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1733584437737 2024-12-07T15:14:01,489 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting bbff0d0c53ec4254bb8dc141577facb3, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=467, earliestPutTs=1733584439027 2024-12-07T15:14:01,490 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting e05e1992037a4b8a87b707918eb3f4f3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=452, earliestPutTs=1733584437855 2024-12-07T15:14:01,490 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 8104550a8c0941bcb193e3d73aa3abbe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=467, earliestPutTs=1733584439042 2024-12-07T15:14:01,503 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 53401f0681ce510d042c1599ea9bc7e2#A#compaction#95 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:14:01,504 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/A/40f15369ca8e44b49d8a01bcf2b43902 is 50, key is test_row_0/A:col10/1733584440196/Put/seqid=0 2024-12-07T15:14:01,507 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 53401f0681ce510d042c1599ea9bc7e2#B#compaction#96 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:14:01,507 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/B/36ffd919559549bbbc859cdefb6a41a9 is 50, key is test_row_0/B:col10/1733584440196/Put/seqid=0 2024-12-07T15:14:01,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741932_1108 (size=13391) 2024-12-07T15:14:01,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741933_1109 (size=13391) 2024-12-07T15:14:01,534 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/A/40f15369ca8e44b49d8a01bcf2b43902 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/40f15369ca8e44b49d8a01bcf2b43902 2024-12-07T15:14:01,535 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/B/36ffd919559549bbbc859cdefb6a41a9 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/36ffd919559549bbbc859cdefb6a41a9 2024-12-07T15:14:01,543 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 53401f0681ce510d042c1599ea9bc7e2/A of 53401f0681ce510d042c1599ea9bc7e2 into 40f15369ca8e44b49d8a01bcf2b43902(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:14:01,543 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 53401f0681ce510d042c1599ea9bc7e2: 2024-12-07T15:14:01,544 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., storeName=53401f0681ce510d042c1599ea9bc7e2/A, priority=13, startTime=1733584441485; duration=0sec 2024-12-07T15:14:01,544 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T15:14:01,544 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 53401f0681ce510d042c1599ea9bc7e2:A 2024-12-07T15:14:01,544 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-07T15:14:01,548 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 53401f0681ce510d042c1599ea9bc7e2/B of 53401f0681ce510d042c1599ea9bc7e2 into 36ffd919559549bbbc859cdefb6a41a9(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:14:01,548 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 53401f0681ce510d042c1599ea9bc7e2: 2024-12-07T15:14:01,548 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., storeName=53401f0681ce510d042c1599ea9bc7e2/B, priority=13, startTime=1733584441486; duration=0sec 2024-12-07T15:14:01,548 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:14:01,548 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 53401f0681ce510d042c1599ea9bc7e2:B 2024-12-07T15:14:01,549 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50124 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-07T15:14:01,549 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1541): 53401f0681ce510d042c1599ea9bc7e2/C is initiating minor compaction (all files) 2024-12-07T15:14:01,550 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 53401f0681ce510d042c1599ea9bc7e2/C in TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:14:01,550 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/4262fec1d81f4ee9846791a37a2402cf, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/2fa0e85e039f40f69e0128098049e403, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/e6cbb92d0359454da666ec4ce16bf59b, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/e96dacee72864a998b77a09410a09df1] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp, totalSize=48.9 K 2024-12-07T15:14:01,550 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4262fec1d81f4ee9846791a37a2402cf, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1733584437074 2024-12-07T15:14:01,551 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2fa0e85e039f40f69e0128098049e403, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1733584437737 2024-12-07T15:14:01,551 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6cbb92d0359454da666ec4ce16bf59b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=452, earliestPutTs=1733584437855 2024-12-07T15:14:01,552 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting e96dacee72864a998b77a09410a09df1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=467, earliestPutTs=1733584439042 2024-12-07T15:14:01,566 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 53401f0681ce510d042c1599ea9bc7e2#C#compaction#97 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:14:01,567 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/C/0c78aed1f3d74990bf3724a372f93fe4 is 50, key is test_row_0/C:col10/1733584440196/Put/seqid=0 2024-12-07T15:14:01,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741934_1110 (size=13357) 2024-12-07T15:14:01,994 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/C/0c78aed1f3d74990bf3724a372f93fe4 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/0c78aed1f3d74990bf3724a372f93fe4 2024-12-07T15:14:02,001 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 53401f0681ce510d042c1599ea9bc7e2/C of 53401f0681ce510d042c1599ea9bc7e2 into 0c78aed1f3d74990bf3724a372f93fe4(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:14:02,001 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 53401f0681ce510d042c1599ea9bc7e2: 2024-12-07T15:14:02,001 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., storeName=53401f0681ce510d042c1599ea9bc7e2/C, priority=12, startTime=1733584441486; duration=0sec 2024-12-07T15:14:02,001 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:14:02,001 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 53401f0681ce510d042c1599ea9bc7e2:C 2024-12-07T15:14:02,010 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-12-07T15:14:02,010 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T15:14:02,011 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-12-07T15:14:02,011 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T15:14:02,013 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-12-07T15:14:02,013 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T15:14:02,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-07T15:14:02,014 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T15:14:02,016 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:14:02,016 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:14:02,016 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:14:02,017 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:14:02,017 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:14:02,017 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T15:14:02,017 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:14:02,017 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T15:14:02,017 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T15:14:02,018 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-12-07T15:14:02,018 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T15:14:02,018 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:14:02,018 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:14:02,018 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T15:14:02,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on 53401f0681ce510d042c1599ea9bc7e2 2024-12-07T15:14:02,065 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 53401f0681ce510d042c1599ea9bc7e2 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-07T15:14:02,065 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-12-07T15:14:02,065 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T15:14:02,065 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:14:02,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 53401f0681ce510d042c1599ea9bc7e2, store=A 2024-12-07T15:14:02,066 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:14:02,066 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T15:14:02,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:14:02,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 53401f0681ce510d042c1599ea9bc7e2, store=B 2024-12-07T15:14:02,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:14:02,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 53401f0681ce510d042c1599ea9bc7e2, store=C 2024-12-07T15:14:02,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:14:02,074 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/A/ff05e21260134b8da9ce22b511f3539a is 50, key is test_row_0/A:col10/1733584442063/Put/seqid=0 2024-12-07T15:14:02,075 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-12-07T15:14:02,075 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T15:14:02,075 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:14:02,075 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:14:02,076 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T15:14:02,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741935_1111 (size=12301) 2024-12-07T15:14:02,079 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=490 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/A/ff05e21260134b8da9ce22b511f3539a 2024-12-07T15:14:02,086 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-12-07T15:14:02,086 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T15:14:02,086 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:14:02,086 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:14:02,086 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T15:14:02,088 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/B/ae47332e3301466790f170ff6463d82d is 50, key is test_row_0/B:col10/1733584442063/Put/seqid=0 2024-12-07T15:14:02,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741936_1112 (size=12301) 2024-12-07T15:14:02,396 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-12-07T15:14:02,396 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-12-07T15:14:02,396 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T15:14:02,396 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T15:14:02,396 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:14:02,396 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:14:02,396 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:14:02,396 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:14:02,396 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T15:14:02,396 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-07T15:14:02,396 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 19 2024-12-07T15:14:02,396 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T15:14:02,396 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 164 2024-12-07T15:14:02,396 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 17 2024-12-07T15:14:02,396 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 19 2024-12-07T15:14:02,397 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 173 2024-12-07T15:14:02,397 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-07T15:14:02,397 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2780 2024-12-07T15:14:02,397 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2721 2024-12-07T15:14:02,397 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-07T15:14:02,397 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1257 2024-12-07T15:14:02,397 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3770 rows 2024-12-07T15:14:02,397 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1258 2024-12-07T15:14:02,397 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3769 rows 2024-12-07T15:14:02,397 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-07T15:14:02,397 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-07T15:14:02,397 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:14:02,397 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:14:02,397 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T15:14:02,402 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-07T15:14:02,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-07T15:14:02,411 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733584442411"}]},"ts":"1733584442411"} 2024-12-07T15:14:02,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-07T15:14:02,413 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-07T15:14:02,413 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-07T15:14:02,415 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-07T15:14:02,421 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=53401f0681ce510d042c1599ea9bc7e2, UNASSIGN}] 2024-12-07T15:14:02,422 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=53401f0681ce510d042c1599ea9bc7e2, UNASSIGN 2024-12-07T15:14:02,423 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=53401f0681ce510d042c1599ea9bc7e2, regionState=CLOSING, regionLocation=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:02,425 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=53401f0681ce510d042c1599ea9bc7e2, UNASSIGN because future has completed 2024-12-07T15:14:02,426 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-07T15:14:02,426 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure 53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401}] 2024-12-07T15:14:02,508 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=490 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/B/ae47332e3301466790f170ff6463d82d 2024-12-07T15:14:02,517 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/C/aca6c701958c4f2b80a845b1b960147c is 50, key is test_row_0/C:col10/1733584442063/Put/seqid=0 2024-12-07T15:14:02,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741937_1113 (size=12301) 2024-12-07T15:14:02,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-07T15:14:02,584 INFO [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(122): Close 53401f0681ce510d042c1599ea9bc7e2 2024-12-07T15:14:02,584 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-07T15:14:02,585 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1722): Closing 53401f0681ce510d042c1599ea9bc7e2, disabling compactions & flushes 2024-12-07T15:14:02,585 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:14:02,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-07T15:14:02,921 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=490 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/C/aca6c701958c4f2b80a845b1b960147c 2024-12-07T15:14:02,927 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/A/ff05e21260134b8da9ce22b511f3539a as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/ff05e21260134b8da9ce22b511f3539a 2024-12-07T15:14:02,931 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/ff05e21260134b8da9ce22b511f3539a, entries=150, sequenceid=490, filesize=12.0 K 2024-12-07T15:14:02,932 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/B/ae47332e3301466790f170ff6463d82d as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/ae47332e3301466790f170ff6463d82d 2024-12-07T15:14:02,936 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/ae47332e3301466790f170ff6463d82d, entries=150, sequenceid=490, filesize=12.0 K 2024-12-07T15:14:02,937 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/C/aca6c701958c4f2b80a845b1b960147c as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/aca6c701958c4f2b80a845b1b960147c 2024-12-07T15:14:02,942 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/aca6c701958c4f2b80a845b1b960147c, entries=150, sequenceid=490, filesize=12.0 K 2024-12-07T15:14:02,943 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=26.84 KB/27480 for 53401f0681ce510d042c1599ea9bc7e2 in 878ms, sequenceid=490, compaction requested=false 2024-12-07T15:14:02,943 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 53401f0681ce510d042c1599ea9bc7e2: 2024-12-07T15:14:02,943 INFO [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:14:02,943 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:14:02,943 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. after waiting 0 ms 2024-12-07T15:14:02,943 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:14:02,943 INFO [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(2902): Flushing 53401f0681ce510d042c1599ea9bc7e2 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-12-07T15:14:02,943 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 53401f0681ce510d042c1599ea9bc7e2, store=A 2024-12-07T15:14:02,943 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:14:02,943 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 53401f0681ce510d042c1599ea9bc7e2, store=B 2024-12-07T15:14:02,944 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:14:02,944 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 53401f0681ce510d042c1599ea9bc7e2, store=C 2024-12-07T15:14:02,944 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:14:02,948 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/A/b3504acc4b40499fb72cea8792081685 is 50, key is test_row_0/A:col10/1733584442394/Put/seqid=0 2024-12-07T15:14:02,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741938_1114 (size=9857) 2024-12-07T15:14:03,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-07T15:14:03,359 INFO [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=497 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/A/b3504acc4b40499fb72cea8792081685 2024-12-07T15:14:03,367 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/B/7360fa88901a446d95e09d83109ec22d is 50, key is test_row_0/B:col10/1733584442394/Put/seqid=0 2024-12-07T15:14:03,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741939_1115 (size=9857) 2024-12-07T15:14:03,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-07T15:14:03,771 INFO [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=497 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/B/7360fa88901a446d95e09d83109ec22d 2024-12-07T15:14:03,780 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/C/bc87b24e418a47fb9d7117745f1e9a05 is 50, key is test_row_0/C:col10/1733584442394/Put/seqid=0 2024-12-07T15:14:03,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741940_1116 (size=9857) 2024-12-07T15:14:04,187 INFO [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=497 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/C/bc87b24e418a47fb9d7117745f1e9a05 2024-12-07T15:14:04,192 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/A/b3504acc4b40499fb72cea8792081685 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/b3504acc4b40499fb72cea8792081685 2024-12-07T15:14:04,196 INFO [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/b3504acc4b40499fb72cea8792081685, entries=100, sequenceid=497, filesize=9.6 K 2024-12-07T15:14:04,196 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/B/7360fa88901a446d95e09d83109ec22d as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/7360fa88901a446d95e09d83109ec22d 2024-12-07T15:14:04,200 INFO [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/7360fa88901a446d95e09d83109ec22d, entries=100, sequenceid=497, filesize=9.6 K 2024-12-07T15:14:04,201 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/.tmp/C/bc87b24e418a47fb9d7117745f1e9a05 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/bc87b24e418a47fb9d7117745f1e9a05 2024-12-07T15:14:04,205 INFO [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/bc87b24e418a47fb9d7117745f1e9a05, entries=100, sequenceid=497, filesize=9.6 K 2024-12-07T15:14:04,206 INFO [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 53401f0681ce510d042c1599ea9bc7e2 in 1263ms, sequenceid=497, compaction requested=true 2024-12-07T15:14:04,207 DEBUG [StoreCloser-TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/44e00011a9ee46eaa3b997fac29d50c7, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/095c5365ce6d4c43917491448d0870ac, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/beb5d366a81b4fbc81569bbea20bdfb5, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/2fa6c00101fd417caa3eb8ac1c5b7395, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/9fd1b9703f8d435993175c04de4066a3, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/2cb709721cbd4613bf10a9064351a369, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/a072b7d4a5c249a7828f3bec3ca4bf21, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/1e21c007e21147dbaf96246b1c5d2b78, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/89cd7191e60346948ebc9cd6b71d587a, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/e67d1910a4a84db7b30984719ae004b4, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/bab8cfd5cb0949ac887ba3dafceb217e, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/f0590943cc9b4ff88ac3631deecf9cb9, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/cf8f68bb629d4426af4882827e61212d, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/2dfa6e77cfd54c38af89ad1c4eea550a, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/1c71939fdabf454394af03e5acc43921, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/9859a436fdc54dbe94857cbbac4ce62a, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/28b9003d663242d1bea9c6e7d20b3f99, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/4f37ced65c3e40dd85afb3c068a8a50a, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/7ca55ff415ce4f1cb587a3a6484025de, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/9dc7ea62c09f426d931bc23d362be474, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/6916c638b3d74ed8941c157beefb47f9, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/eacf947c67264a109c5ccceb000c3bca, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/4a12ef7335bc44a1873dd4f8b79e2e22, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/f7990215a0ca4783b0d9eac1aa73279e, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/58a57450de9b4228b69085609e8a811f, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/4ab84ea3fe914664be05058468299eed, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/05da278fb6274982a29989f83cf28a6a, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/f401219dffb941d798f02420c82d5be7, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/f37911ada0f04e0f9514fee229ea1d14, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/bf77e1f312814f06b3fb99b44425174e, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/67162cecbba8410c99abf042d4869aa0, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/bbff0d0c53ec4254bb8dc141577facb3] to archive 2024-12-07T15:14:04,211 DEBUG [StoreCloser-TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-07T15:14:04,216 DEBUG [StoreCloser-TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/44e00011a9ee46eaa3b997fac29d50c7 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/44e00011a9ee46eaa3b997fac29d50c7 2024-12-07T15:14:04,218 DEBUG [StoreCloser-TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/095c5365ce6d4c43917491448d0870ac to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/095c5365ce6d4c43917491448d0870ac 2024-12-07T15:14:04,219 DEBUG [StoreCloser-TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/beb5d366a81b4fbc81569bbea20bdfb5 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/beb5d366a81b4fbc81569bbea20bdfb5 2024-12-07T15:14:04,220 DEBUG [StoreCloser-TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/2fa6c00101fd417caa3eb8ac1c5b7395 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/2fa6c00101fd417caa3eb8ac1c5b7395 2024-12-07T15:14:04,221 DEBUG [StoreCloser-TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/9fd1b9703f8d435993175c04de4066a3 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/9fd1b9703f8d435993175c04de4066a3 2024-12-07T15:14:04,223 DEBUG [StoreCloser-TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/2cb709721cbd4613bf10a9064351a369 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/2cb709721cbd4613bf10a9064351a369 2024-12-07T15:14:04,225 DEBUG [StoreCloser-TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/a072b7d4a5c249a7828f3bec3ca4bf21 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/a072b7d4a5c249a7828f3bec3ca4bf21 2024-12-07T15:14:04,226 DEBUG [StoreCloser-TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/1e21c007e21147dbaf96246b1c5d2b78 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/1e21c007e21147dbaf96246b1c5d2b78 2024-12-07T15:14:04,227 DEBUG [StoreCloser-TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/89cd7191e60346948ebc9cd6b71d587a to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/89cd7191e60346948ebc9cd6b71d587a 2024-12-07T15:14:04,229 DEBUG [StoreCloser-TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/e67d1910a4a84db7b30984719ae004b4 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/e67d1910a4a84db7b30984719ae004b4 2024-12-07T15:14:04,230 DEBUG [StoreCloser-TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/bab8cfd5cb0949ac887ba3dafceb217e to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/bab8cfd5cb0949ac887ba3dafceb217e 2024-12-07T15:14:04,231 DEBUG [StoreCloser-TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/f0590943cc9b4ff88ac3631deecf9cb9 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/f0590943cc9b4ff88ac3631deecf9cb9 2024-12-07T15:14:04,233 DEBUG [StoreCloser-TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/cf8f68bb629d4426af4882827e61212d to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/cf8f68bb629d4426af4882827e61212d 2024-12-07T15:14:04,234 DEBUG [StoreCloser-TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/2dfa6e77cfd54c38af89ad1c4eea550a to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/2dfa6e77cfd54c38af89ad1c4eea550a 2024-12-07T15:14:04,235 DEBUG [StoreCloser-TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/1c71939fdabf454394af03e5acc43921 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/1c71939fdabf454394af03e5acc43921 2024-12-07T15:14:04,237 DEBUG [StoreCloser-TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/9859a436fdc54dbe94857cbbac4ce62a to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/9859a436fdc54dbe94857cbbac4ce62a 2024-12-07T15:14:04,238 DEBUG [StoreCloser-TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/28b9003d663242d1bea9c6e7d20b3f99 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/28b9003d663242d1bea9c6e7d20b3f99 2024-12-07T15:14:04,239 DEBUG [StoreCloser-TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/4f37ced65c3e40dd85afb3c068a8a50a to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/4f37ced65c3e40dd85afb3c068a8a50a 2024-12-07T15:14:04,241 DEBUG [StoreCloser-TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/7ca55ff415ce4f1cb587a3a6484025de to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/7ca55ff415ce4f1cb587a3a6484025de 2024-12-07T15:14:04,242 DEBUG [StoreCloser-TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/9dc7ea62c09f426d931bc23d362be474 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/9dc7ea62c09f426d931bc23d362be474 2024-12-07T15:14:04,243 DEBUG [StoreCloser-TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/6916c638b3d74ed8941c157beefb47f9 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/6916c638b3d74ed8941c157beefb47f9 2024-12-07T15:14:04,244 DEBUG [StoreCloser-TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/eacf947c67264a109c5ccceb000c3bca to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/eacf947c67264a109c5ccceb000c3bca 2024-12-07T15:14:04,246 DEBUG [StoreCloser-TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/4a12ef7335bc44a1873dd4f8b79e2e22 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/4a12ef7335bc44a1873dd4f8b79e2e22 2024-12-07T15:14:04,247 DEBUG [StoreCloser-TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/f7990215a0ca4783b0d9eac1aa73279e to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/f7990215a0ca4783b0d9eac1aa73279e 2024-12-07T15:14:04,249 DEBUG [StoreCloser-TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/58a57450de9b4228b69085609e8a811f to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/58a57450de9b4228b69085609e8a811f 2024-12-07T15:14:04,250 DEBUG [StoreCloser-TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/4ab84ea3fe914664be05058468299eed to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/4ab84ea3fe914664be05058468299eed 2024-12-07T15:14:04,252 DEBUG [StoreCloser-TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/05da278fb6274982a29989f83cf28a6a to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/05da278fb6274982a29989f83cf28a6a 2024-12-07T15:14:04,253 DEBUG [StoreCloser-TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/f401219dffb941d798f02420c82d5be7 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/f401219dffb941d798f02420c82d5be7 2024-12-07T15:14:04,254 DEBUG [StoreCloser-TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/f37911ada0f04e0f9514fee229ea1d14 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/f37911ada0f04e0f9514fee229ea1d14 2024-12-07T15:14:04,255 DEBUG [StoreCloser-TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/bf77e1f312814f06b3fb99b44425174e to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/bf77e1f312814f06b3fb99b44425174e 2024-12-07T15:14:04,257 DEBUG [StoreCloser-TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/67162cecbba8410c99abf042d4869aa0 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/67162cecbba8410c99abf042d4869aa0 2024-12-07T15:14:04,258 DEBUG [StoreCloser-TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/bbff0d0c53ec4254bb8dc141577facb3 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/bbff0d0c53ec4254bb8dc141577facb3 2024-12-07T15:14:04,272 DEBUG [StoreCloser-TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/05474634a257474ca595ea1ca3c4a52c, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/321b66df157743c98b652dc7f6b6c891, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/159141ec8899440bbab8fb159d74aa53, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/d4b8a070cbb848ca88d3f3284c729089, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/7d5862eb5798423e9fe79bde85746d66, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/826837eaedcb40dda69be772defc0c0a, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/e6221ebb109f42729c4b05800d1f65a2, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/0dd77d70596245cf91b2cd01af0e9596, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/42d85a580a414589a2eb4a55b248f749, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/c164145fd7cc40358f7627dcf9d655e2, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/fc74713e99af49be9a1c3fb3f4b15798, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/939c50a7356a485eabad8900a6142f61, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/e3f19ea1e55344999bce8c2d389200fa, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/1744017ec4b0452eb570b225b4016f57, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/12634d9acca74457b3ebd2647f887899, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/b1399a8d338046ce988ea72cfb3ab3e8, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/614e2024813945019a2586f3551f080b, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/17528d69805e4b7bac291b5b269acfc2, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/bb9533654c7c49c9809659effedd3a3a, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/dd640f8ad6d9483cad6b4bdad1827b9b, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/76302b6cebb845269b72194877e1f507, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/2795874958f646d69ab7fe486e2e969c, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/e479e6482dda4be8ad710c95097a11e2, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/7d2e9e37dc504eeb9565c8ad6bb9ef25, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/d3687378c6044ca9a27c3aa901642194, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/21d198e4d0f44c5faa703daecf3e80ef, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/6cd47374edb144478cfef41ba51bf1a1, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/9341a047f50945deb2f8b52293339f54, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/8fc90821f61c4378a27b717e06c4a337, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/ad1fa4ed689e493e9e3b0442fb909386, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/e05e1992037a4b8a87b707918eb3f4f3, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/8104550a8c0941bcb193e3d73aa3abbe] to archive 2024-12-07T15:14:04,273 DEBUG [StoreCloser-TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-07T15:14:04,274 DEBUG [StoreCloser-TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/05474634a257474ca595ea1ca3c4a52c to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/05474634a257474ca595ea1ca3c4a52c 2024-12-07T15:14:04,275 DEBUG [StoreCloser-TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/321b66df157743c98b652dc7f6b6c891 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/321b66df157743c98b652dc7f6b6c891 2024-12-07T15:14:04,277 DEBUG [StoreCloser-TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/159141ec8899440bbab8fb159d74aa53 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/159141ec8899440bbab8fb159d74aa53 2024-12-07T15:14:04,278 DEBUG [StoreCloser-TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/d4b8a070cbb848ca88d3f3284c729089 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/d4b8a070cbb848ca88d3f3284c729089 2024-12-07T15:14:04,279 DEBUG [StoreCloser-TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/7d5862eb5798423e9fe79bde85746d66 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/7d5862eb5798423e9fe79bde85746d66 2024-12-07T15:14:04,280 DEBUG [StoreCloser-TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/826837eaedcb40dda69be772defc0c0a to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/826837eaedcb40dda69be772defc0c0a 2024-12-07T15:14:04,281 DEBUG [StoreCloser-TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/e6221ebb109f42729c4b05800d1f65a2 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/e6221ebb109f42729c4b05800d1f65a2 2024-12-07T15:14:04,283 DEBUG [StoreCloser-TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/0dd77d70596245cf91b2cd01af0e9596 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/0dd77d70596245cf91b2cd01af0e9596 2024-12-07T15:14:04,284 DEBUG [StoreCloser-TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/42d85a580a414589a2eb4a55b248f749 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/42d85a580a414589a2eb4a55b248f749 2024-12-07T15:14:04,285 DEBUG [StoreCloser-TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/c164145fd7cc40358f7627dcf9d655e2 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/c164145fd7cc40358f7627dcf9d655e2 2024-12-07T15:14:04,286 DEBUG [StoreCloser-TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/fc74713e99af49be9a1c3fb3f4b15798 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/fc74713e99af49be9a1c3fb3f4b15798 2024-12-07T15:14:04,288 DEBUG [StoreCloser-TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/939c50a7356a485eabad8900a6142f61 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/939c50a7356a485eabad8900a6142f61 2024-12-07T15:14:04,289 DEBUG [StoreCloser-TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/e3f19ea1e55344999bce8c2d389200fa to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/e3f19ea1e55344999bce8c2d389200fa 2024-12-07T15:14:04,290 DEBUG [StoreCloser-TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/1744017ec4b0452eb570b225b4016f57 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/1744017ec4b0452eb570b225b4016f57 2024-12-07T15:14:04,291 DEBUG [StoreCloser-TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/12634d9acca74457b3ebd2647f887899 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/12634d9acca74457b3ebd2647f887899 2024-12-07T15:14:04,292 DEBUG [StoreCloser-TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/b1399a8d338046ce988ea72cfb3ab3e8 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/b1399a8d338046ce988ea72cfb3ab3e8 2024-12-07T15:14:04,293 DEBUG [StoreCloser-TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/614e2024813945019a2586f3551f080b to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/614e2024813945019a2586f3551f080b 2024-12-07T15:14:04,295 DEBUG [StoreCloser-TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/17528d69805e4b7bac291b5b269acfc2 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/17528d69805e4b7bac291b5b269acfc2 2024-12-07T15:14:04,296 DEBUG [StoreCloser-TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/bb9533654c7c49c9809659effedd3a3a to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/bb9533654c7c49c9809659effedd3a3a 2024-12-07T15:14:04,297 DEBUG [StoreCloser-TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/dd640f8ad6d9483cad6b4bdad1827b9b to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/dd640f8ad6d9483cad6b4bdad1827b9b 2024-12-07T15:14:04,298 DEBUG [StoreCloser-TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/76302b6cebb845269b72194877e1f507 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/76302b6cebb845269b72194877e1f507 2024-12-07T15:14:04,299 DEBUG [StoreCloser-TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/2795874958f646d69ab7fe486e2e969c to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/2795874958f646d69ab7fe486e2e969c 2024-12-07T15:14:04,301 DEBUG [StoreCloser-TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/e479e6482dda4be8ad710c95097a11e2 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/e479e6482dda4be8ad710c95097a11e2 2024-12-07T15:14:04,302 DEBUG [StoreCloser-TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/7d2e9e37dc504eeb9565c8ad6bb9ef25 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/7d2e9e37dc504eeb9565c8ad6bb9ef25 2024-12-07T15:14:04,303 DEBUG [StoreCloser-TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/d3687378c6044ca9a27c3aa901642194 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/d3687378c6044ca9a27c3aa901642194 2024-12-07T15:14:04,305 DEBUG [StoreCloser-TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/21d198e4d0f44c5faa703daecf3e80ef to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/21d198e4d0f44c5faa703daecf3e80ef 2024-12-07T15:14:04,306 DEBUG [StoreCloser-TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/6cd47374edb144478cfef41ba51bf1a1 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/6cd47374edb144478cfef41ba51bf1a1 2024-12-07T15:14:04,307 DEBUG [StoreCloser-TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/9341a047f50945deb2f8b52293339f54 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/9341a047f50945deb2f8b52293339f54 2024-12-07T15:14:04,308 DEBUG [StoreCloser-TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/8fc90821f61c4378a27b717e06c4a337 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/8fc90821f61c4378a27b717e06c4a337 2024-12-07T15:14:04,309 DEBUG [StoreCloser-TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/ad1fa4ed689e493e9e3b0442fb909386 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/ad1fa4ed689e493e9e3b0442fb909386 2024-12-07T15:14:04,311 DEBUG [StoreCloser-TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/e05e1992037a4b8a87b707918eb3f4f3 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/e05e1992037a4b8a87b707918eb3f4f3 2024-12-07T15:14:04,312 DEBUG [StoreCloser-TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/8104550a8c0941bcb193e3d73aa3abbe to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/8104550a8c0941bcb193e3d73aa3abbe 2024-12-07T15:14:04,314 DEBUG [StoreCloser-TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/a4f19d5c96b446aea3b5af6db256d1f6, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/23ead6ba94dc4c7abdb7c471bb08ed4c, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/3e33d751ad924ded995c3dad74cf5e05, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/27890a55df4445a4bcbaba4fe9ab60ba, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/0d0b65f94e1f4a9dbaacd46759426d55, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/4fa4df3f7c02452fbdcee3e338c8f2dd, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/b2c647e6d0d249dcb1f59ddc8c022ced, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/d8fe0170e6d8410084f1fdf573091f8a, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/55b743c9a9064bd3950d11953a239329, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/6dc0ef423acb42d7995e4ab54623c53e, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/b76cd182600f47e29d58b3b617198a8b, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/1658e68434e242f9bd559456ad36574d, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/a04be5ad74c64d208927c5b0b6ea8e79, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/e067375b409d4e38b8af5dc0b70a5670, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/906c00c7eccd43dcb39afc58bbc5a183, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/de3f5403c83b4aec8338ea7d0c3fa7e2, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/321d9470b4c54af391a5497ce8d5a3df, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/c7b6b1f11237452092223227307bb527, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/7b446d9de80c4b18bba71d6996067c8b, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/87d27ff4e6764394b111eee75e3daf8a, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/d99cbc6ca10b45e7a1129c2097b33c8d, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/c0f918b435074fe3b366dd0923e81539, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/cf8a019e31b948adbd5eb210b0d8f975, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/369e0a4b3ede47369ec5f0c65257637f, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/7e1c3628292e485fbf9a3fe35a198bfe, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/1f5eaa5ea0b2475993f52c0070bb06a9, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/4262fec1d81f4ee9846791a37a2402cf, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/ab8a7cde8a6c457496f39f137cec83cd, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/2fa0e85e039f40f69e0128098049e403, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/e6cbb92d0359454da666ec4ce16bf59b, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/e96dacee72864a998b77a09410a09df1] to archive 2024-12-07T15:14:04,315 DEBUG [StoreCloser-TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-07T15:14:04,317 DEBUG [StoreCloser-TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/a4f19d5c96b446aea3b5af6db256d1f6 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/a4f19d5c96b446aea3b5af6db256d1f6 2024-12-07T15:14:04,318 DEBUG [StoreCloser-TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/23ead6ba94dc4c7abdb7c471bb08ed4c to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/23ead6ba94dc4c7abdb7c471bb08ed4c 2024-12-07T15:14:04,319 DEBUG [StoreCloser-TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/3e33d751ad924ded995c3dad74cf5e05 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/3e33d751ad924ded995c3dad74cf5e05 2024-12-07T15:14:04,320 DEBUG [StoreCloser-TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/27890a55df4445a4bcbaba4fe9ab60ba to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/27890a55df4445a4bcbaba4fe9ab60ba 2024-12-07T15:14:04,321 DEBUG [StoreCloser-TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/0d0b65f94e1f4a9dbaacd46759426d55 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/0d0b65f94e1f4a9dbaacd46759426d55 2024-12-07T15:14:04,322 DEBUG [StoreCloser-TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/4fa4df3f7c02452fbdcee3e338c8f2dd to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/4fa4df3f7c02452fbdcee3e338c8f2dd 2024-12-07T15:14:04,324 DEBUG [StoreCloser-TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/b2c647e6d0d249dcb1f59ddc8c022ced to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/b2c647e6d0d249dcb1f59ddc8c022ced 2024-12-07T15:14:04,325 DEBUG [StoreCloser-TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/d8fe0170e6d8410084f1fdf573091f8a to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/d8fe0170e6d8410084f1fdf573091f8a 2024-12-07T15:14:04,326 DEBUG [StoreCloser-TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/55b743c9a9064bd3950d11953a239329 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/55b743c9a9064bd3950d11953a239329 2024-12-07T15:14:04,327 DEBUG [StoreCloser-TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/6dc0ef423acb42d7995e4ab54623c53e to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/6dc0ef423acb42d7995e4ab54623c53e 2024-12-07T15:14:04,328 DEBUG [StoreCloser-TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/b76cd182600f47e29d58b3b617198a8b to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/b76cd182600f47e29d58b3b617198a8b 2024-12-07T15:14:04,329 DEBUG [StoreCloser-TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/1658e68434e242f9bd559456ad36574d to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/1658e68434e242f9bd559456ad36574d 2024-12-07T15:14:04,330 DEBUG [StoreCloser-TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/a04be5ad74c64d208927c5b0b6ea8e79 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/a04be5ad74c64d208927c5b0b6ea8e79 2024-12-07T15:14:04,331 DEBUG [StoreCloser-TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/e067375b409d4e38b8af5dc0b70a5670 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/e067375b409d4e38b8af5dc0b70a5670 2024-12-07T15:14:04,332 DEBUG [StoreCloser-TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/906c00c7eccd43dcb39afc58bbc5a183 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/906c00c7eccd43dcb39afc58bbc5a183 2024-12-07T15:14:04,333 DEBUG [StoreCloser-TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/de3f5403c83b4aec8338ea7d0c3fa7e2 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/de3f5403c83b4aec8338ea7d0c3fa7e2 2024-12-07T15:14:04,334 DEBUG [StoreCloser-TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/321d9470b4c54af391a5497ce8d5a3df to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/321d9470b4c54af391a5497ce8d5a3df 2024-12-07T15:14:04,335 DEBUG [StoreCloser-TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/c7b6b1f11237452092223227307bb527 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/c7b6b1f11237452092223227307bb527 2024-12-07T15:14:04,336 DEBUG [StoreCloser-TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/7b446d9de80c4b18bba71d6996067c8b to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/7b446d9de80c4b18bba71d6996067c8b 2024-12-07T15:14:04,337 DEBUG [StoreCloser-TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/87d27ff4e6764394b111eee75e3daf8a to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/87d27ff4e6764394b111eee75e3daf8a 2024-12-07T15:14:04,339 DEBUG [StoreCloser-TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/d99cbc6ca10b45e7a1129c2097b33c8d to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/d99cbc6ca10b45e7a1129c2097b33c8d 2024-12-07T15:14:04,340 DEBUG [StoreCloser-TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/c0f918b435074fe3b366dd0923e81539 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/c0f918b435074fe3b366dd0923e81539 2024-12-07T15:14:04,342 DEBUG [StoreCloser-TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/cf8a019e31b948adbd5eb210b0d8f975 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/cf8a019e31b948adbd5eb210b0d8f975 2024-12-07T15:14:04,344 DEBUG [StoreCloser-TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/369e0a4b3ede47369ec5f0c65257637f to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/369e0a4b3ede47369ec5f0c65257637f 2024-12-07T15:14:04,345 DEBUG [StoreCloser-TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/7e1c3628292e485fbf9a3fe35a198bfe to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/7e1c3628292e485fbf9a3fe35a198bfe 2024-12-07T15:14:04,346 DEBUG [StoreCloser-TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/1f5eaa5ea0b2475993f52c0070bb06a9 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/1f5eaa5ea0b2475993f52c0070bb06a9 2024-12-07T15:14:04,347 DEBUG [StoreCloser-TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/4262fec1d81f4ee9846791a37a2402cf to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/4262fec1d81f4ee9846791a37a2402cf 2024-12-07T15:14:04,349 DEBUG [StoreCloser-TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/ab8a7cde8a6c457496f39f137cec83cd to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/ab8a7cde8a6c457496f39f137cec83cd 2024-12-07T15:14:04,350 DEBUG [StoreCloser-TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/2fa0e85e039f40f69e0128098049e403 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/2fa0e85e039f40f69e0128098049e403 2024-12-07T15:14:04,351 DEBUG [StoreCloser-TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/e6cbb92d0359454da666ec4ce16bf59b to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/e6cbb92d0359454da666ec4ce16bf59b 2024-12-07T15:14:04,352 DEBUG [StoreCloser-TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/e96dacee72864a998b77a09410a09df1 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/e96dacee72864a998b77a09410a09df1 2024-12-07T15:14:04,358 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/recovered.edits/500.seqid, newMaxSeqId=500, maxSeqId=1 2024-12-07T15:14:04,359 INFO [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. 2024-12-07T15:14:04,360 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1676): Region close journal for 53401f0681ce510d042c1599ea9bc7e2: Waiting for close lock at 1733584442585Running coprocessor pre-close hooks at 1733584442585Disabling compacts and flushes for region at 1733584442585Disabling writes for close at 1733584442943 (+358 ms)Obtaining lock to block concurrent updates at 1733584442943Preparing flush snapshotting stores in 53401f0681ce510d042c1599ea9bc7e2 at 1733584442943Finished memstore snapshotting TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1733584442944 (+1 ms)Flushing stores of TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2. at 1733584442944Flushing 53401f0681ce510d042c1599ea9bc7e2/A: creating writer at 1733584442944Flushing 53401f0681ce510d042c1599ea9bc7e2/A: appending metadata at 1733584442947 (+3 ms)Flushing 53401f0681ce510d042c1599ea9bc7e2/A: closing flushed file at 1733584442947Flushing 53401f0681ce510d042c1599ea9bc7e2/B: creating writer at 1733584443364 (+417 ms)Flushing 53401f0681ce510d042c1599ea9bc7e2/B: appending metadata at 1733584443366 (+2 ms)Flushing 53401f0681ce510d042c1599ea9bc7e2/B: closing flushed file at 1733584443366Flushing 53401f0681ce510d042c1599ea9bc7e2/C: creating writer at 1733584443776 (+410 ms)Flushing 53401f0681ce510d042c1599ea9bc7e2/C: appending metadata at 1733584443779 (+3 ms)Flushing 53401f0681ce510d042c1599ea9bc7e2/C: closing flushed file at 1733584443779Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@461baa31: reopening flushed file at 1733584444191 (+412 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@12529557: reopening flushed file at 1733584444196 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@544c5c1e: reopening flushed file at 1733584444200 (+4 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 53401f0681ce510d042c1599ea9bc7e2 in 1263ms, sequenceid=497, compaction requested=true at 1733584444206 (+6 ms)Writing region close event to WAL at 1733584444354 (+148 ms)Running coprocessor post-close hooks at 1733584444359 (+5 ms)Closed at 1733584444359 2024-12-07T15:14:04,362 INFO [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(157): Closed 53401f0681ce510d042c1599ea9bc7e2 2024-12-07T15:14:04,364 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=53401f0681ce510d042c1599ea9bc7e2, regionState=CLOSED 2024-12-07T15:14:04,365 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure 53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 because future has completed 2024-12-07T15:14:04,369 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-12-07T15:14:04,370 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; CloseRegionProcedure 53401f0681ce510d042c1599ea9bc7e2, server=d6797ebe9c52,46009,1733584418401 in 1.9420 sec 2024-12-07T15:14:04,372 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-12-07T15:14:04,372 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=53401f0681ce510d042c1599ea9bc7e2, UNASSIGN in 1.9490 sec 2024-12-07T15:14:04,375 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-12-07T15:14:04,375 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9580 sec 2024-12-07T15:14:04,377 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733584444376"}]},"ts":"1733584444376"} 2024-12-07T15:14:04,378 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-07T15:14:04,378 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-07T15:14:04,381 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9750 sec 2024-12-07T15:14:04,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-07T15:14:04,563 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-07T15:14:04,566 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-07T15:14:04,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-07T15:14:04,572 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=31, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-07T15:14:04,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-07T15:14:04,573 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=31, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-07T15:14:04,580 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2 2024-12-07T15:14:04,584 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A, FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B, FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C, FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/recovered.edits] 2024-12-07T15:14:04,587 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/40f15369ca8e44b49d8a01bcf2b43902 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/40f15369ca8e44b49d8a01bcf2b43902 2024-12-07T15:14:04,589 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/b3504acc4b40499fb72cea8792081685 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/b3504acc4b40499fb72cea8792081685 2024-12-07T15:14:04,590 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/ff05e21260134b8da9ce22b511f3539a to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/A/ff05e21260134b8da9ce22b511f3539a 2024-12-07T15:14:04,593 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/36ffd919559549bbbc859cdefb6a41a9 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/36ffd919559549bbbc859cdefb6a41a9 2024-12-07T15:14:04,594 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/7360fa88901a446d95e09d83109ec22d to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/7360fa88901a446d95e09d83109ec22d 2024-12-07T15:14:04,595 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/ae47332e3301466790f170ff6463d82d to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/B/ae47332e3301466790f170ff6463d82d 2024-12-07T15:14:04,598 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/0c78aed1f3d74990bf3724a372f93fe4 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/0c78aed1f3d74990bf3724a372f93fe4 2024-12-07T15:14:04,599 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/aca6c701958c4f2b80a845b1b960147c to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/aca6c701958c4f2b80a845b1b960147c 2024-12-07T15:14:04,601 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/bc87b24e418a47fb9d7117745f1e9a05 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/C/bc87b24e418a47fb9d7117745f1e9a05 2024-12-07T15:14:04,604 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/recovered.edits/500.seqid to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2/recovered.edits/500.seqid 2024-12-07T15:14:04,605 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/53401f0681ce510d042c1599ea9bc7e2 2024-12-07T15:14:04,605 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-07T15:14:04,610 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=31, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-07T15:14:04,614 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-12-07T15:14:04,622 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-07T15:14:04,668 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-07T15:14:04,670 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=31, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-07T15:14:04,670 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-07T15:14:04,671 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733584444670"}]},"ts":"9223372036854775807"} 2024-12-07T15:14:04,676 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-07T15:14:04,676 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 53401f0681ce510d042c1599ea9bc7e2, NAME => 'TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2.', STARTKEY => '', ENDKEY => ''}] 2024-12-07T15:14:04,676 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-07T15:14:04,676 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733584444676"}]},"ts":"9223372036854775807"} 2024-12-07T15:14:04,680 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-07T15:14:04,680 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=31, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-07T15:14:04,681 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 113 msec 2024-12-07T15:14:04,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-07T15:14:04,684 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-07T15:14:04,684 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-07T15:14:04,697 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=248 (was 211) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;d6797ebe9c52:46009-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1222216727_22 at /127.0.0.1:49718 [Waiting for operation #320] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2134372496_22 at /127.0.0.1:33458 [Waiting for operation #334] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=453 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=411 (was 255) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7213 (was 7923) 2024-12-07T15:14:04,707 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=248, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=411, ProcessCount=11, AvailableMemoryMB=7211 2024-12-07T15:14:04,709 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-07T15:14:04,709 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-07T15:14:04,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] procedure2.ProcedureExecutor(1139): Stored pid=32, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-07T15:14:04,711 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-07T15:14:04,712 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:04,712 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 32 2024-12-07T15:14:04,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-07T15:14:04,712 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-07T15:14:04,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741941_1117 (size=1061) 2024-12-07T15:14:04,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-07T15:14:05,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-07T15:14:05,121 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd 2024-12-07T15:14:05,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741942_1118 (size=53) 2024-12-07T15:14:05,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-07T15:14:05,528 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T15:14:05,528 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing b393b7989f53c0069cd9db33cd3a7e39, disabling compactions & flushes 2024-12-07T15:14:05,528 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:05,528 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:05,528 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. after waiting 0 ms 2024-12-07T15:14:05,528 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:05,528 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:05,528 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for b393b7989f53c0069cd9db33cd3a7e39: Waiting for close lock at 1733584445528Disabling compacts and flushes for region at 1733584445528Disabling writes for close at 1733584445528Writing region close event to WAL at 1733584445528Closed at 1733584445528 2024-12-07T15:14:05,530 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-07T15:14:05,530 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733584445530"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733584445530"}]},"ts":"1733584445530"} 2024-12-07T15:14:05,532 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-07T15:14:05,534 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-07T15:14:05,534 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733584445534"}]},"ts":"1733584445534"} 2024-12-07T15:14:05,536 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-07T15:14:05,536 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b393b7989f53c0069cd9db33cd3a7e39, ASSIGN}] 2024-12-07T15:14:05,537 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b393b7989f53c0069cd9db33cd3a7e39, ASSIGN 2024-12-07T15:14:05,538 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b393b7989f53c0069cd9db33cd3a7e39, ASSIGN; state=OFFLINE, location=d6797ebe9c52,46009,1733584418401; forceNewPlan=false, retain=false 2024-12-07T15:14:05,689 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=b393b7989f53c0069cd9db33cd3a7e39, regionState=OPENING, regionLocation=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:05,691 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b393b7989f53c0069cd9db33cd3a7e39, ASSIGN because future has completed 2024-12-07T15:14:05,692 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401}] 2024-12-07T15:14:05,848 INFO [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:05,848 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} 2024-12-07T15:14:05,848 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees b393b7989f53c0069cd9db33cd3a7e39 2024-12-07T15:14:05,848 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T15:14:05,849 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for b393b7989f53c0069cd9db33cd3a7e39 2024-12-07T15:14:05,849 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for b393b7989f53c0069cd9db33cd3a7e39 2024-12-07T15:14:05,850 INFO [StoreOpener-b393b7989f53c0069cd9db33cd3a7e39-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region b393b7989f53c0069cd9db33cd3a7e39 2024-12-07T15:14:05,851 INFO [StoreOpener-b393b7989f53c0069cd9db33cd3a7e39-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-07T15:14:05,852 INFO [StoreOpener-b393b7989f53c0069cd9db33cd3a7e39-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b393b7989f53c0069cd9db33cd3a7e39 columnFamilyName A 2024-12-07T15:14:05,853 DEBUG [StoreOpener-b393b7989f53c0069cd9db33cd3a7e39-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:05,853 INFO [StoreOpener-b393b7989f53c0069cd9db33cd3a7e39-1 {}] regionserver.HStore(327): Store=b393b7989f53c0069cd9db33cd3a7e39/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T15:14:05,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-07T15:14:05,853 INFO [StoreOpener-b393b7989f53c0069cd9db33cd3a7e39-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region b393b7989f53c0069cd9db33cd3a7e39 2024-12-07T15:14:05,854 INFO [StoreOpener-b393b7989f53c0069cd9db33cd3a7e39-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-07T15:14:05,855 INFO [StoreOpener-b393b7989f53c0069cd9db33cd3a7e39-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b393b7989f53c0069cd9db33cd3a7e39 columnFamilyName B 2024-12-07T15:14:05,855 DEBUG [StoreOpener-b393b7989f53c0069cd9db33cd3a7e39-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:05,855 INFO [StoreOpener-b393b7989f53c0069cd9db33cd3a7e39-1 {}] regionserver.HStore(327): Store=b393b7989f53c0069cd9db33cd3a7e39/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T15:14:05,855 INFO [StoreOpener-b393b7989f53c0069cd9db33cd3a7e39-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region b393b7989f53c0069cd9db33cd3a7e39 2024-12-07T15:14:05,856 INFO [StoreOpener-b393b7989f53c0069cd9db33cd3a7e39-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-07T15:14:05,857 INFO [StoreOpener-b393b7989f53c0069cd9db33cd3a7e39-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b393b7989f53c0069cd9db33cd3a7e39 columnFamilyName C 2024-12-07T15:14:05,857 DEBUG [StoreOpener-b393b7989f53c0069cd9db33cd3a7e39-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:05,857 INFO [StoreOpener-b393b7989f53c0069cd9db33cd3a7e39-1 {}] regionserver.HStore(327): Store=b393b7989f53c0069cd9db33cd3a7e39/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T15:14:05,857 INFO [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:05,857 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for b393b7989f53c0069cd9db33cd3a7e39 2024-12-07T15:14:05,858 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39 2024-12-07T15:14:05,859 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39 2024-12-07T15:14:05,860 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for b393b7989f53c0069cd9db33cd3a7e39 2024-12-07T15:14:05,860 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for b393b7989f53c0069cd9db33cd3a7e39 2024-12-07T15:14:05,860 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-07T15:14:05,862 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for b393b7989f53c0069cd9db33cd3a7e39 2024-12-07T15:14:05,864 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T15:14:05,865 INFO [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened b393b7989f53c0069cd9db33cd3a7e39; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67481358, jitterRate=0.005550593137741089}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-07T15:14:05,865 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b393b7989f53c0069cd9db33cd3a7e39 2024-12-07T15:14:05,866 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for b393b7989f53c0069cd9db33cd3a7e39: Running coprocessor pre-open hook at 1733584445849Writing region info on filesystem at 1733584445849Initializing all the Stores at 1733584445849Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733584445849Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733584445850 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733584445850Cleaning up temporary data from old regions at 1733584445860 (+10 ms)Running coprocessor post-open hooks at 1733584445865 (+5 ms)Region opened successfully at 1733584445866 (+1 ms) 2024-12-07T15:14:05,867 INFO [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., pid=34, masterSystemTime=1733584445844 2024-12-07T15:14:05,870 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:05,870 INFO [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:05,870 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=b393b7989f53c0069cd9db33cd3a7e39, regionState=OPEN, openSeqNum=2, regionLocation=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:05,872 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 because future has completed 2024-12-07T15:14:05,876 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-12-07T15:14:05,876 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; OpenRegionProcedure b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 in 182 msec 2024-12-07T15:14:05,879 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-12-07T15:14:05,879 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b393b7989f53c0069cd9db33cd3a7e39, ASSIGN in 340 msec 2024-12-07T15:14:05,880 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-07T15:14:05,880 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733584445880"}]},"ts":"1733584445880"} 2024-12-07T15:14:05,881 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-07T15:14:05,882 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-07T15:14:05,884 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1730 sec 2024-12-07T15:14:06,775 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-07T15:14:06,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-07T15:14:06,863 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-07T15:14:06,865 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@736030eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:14:06,865 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6797ebe9c52,36915,-1 for getting cluster id 2024-12-07T15:14:06,865 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T15:14:06,867 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f8956274-ed33-4e16-82eb-d5d5911ec893' 2024-12-07T15:14:06,867 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T15:14:06,867 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f8956274-ed33-4e16-82eb-d5d5911ec893" 2024-12-07T15:14:06,867 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b5ba56d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:14:06,867 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6797ebe9c52,36915,-1] 2024-12-07T15:14:06,868 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T15:14:06,868 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:14:06,869 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33044, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T15:14:06,870 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2766031b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:14:06,870 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T15:14:06,871 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6797ebe9c52,46009,1733584418401, seqNum=-1] 2024-12-07T15:14:06,872 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T15:14:06,873 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37960, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T15:14:06,874 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-07T15:14:06,875 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is d6797ebe9c52,36915,1733584417704 2024-12-07T15:14:06,875 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5b72cf43 2024-12-07T15:14:06,875 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-07T15:14:06,876 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33056, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-07T15:14:06,881 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-07T15:14:06,881 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-07T15:14:06,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] procedure2.ProcedureExecutor(1139): Stored pid=35, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-07T15:14:06,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741943_1119 (size=1097) 2024-12-07T15:14:06,901 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-07T15:14:06,901 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-07T15:14:06,905 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-07T15:14:06,912 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b393b7989f53c0069cd9db33cd3a7e39, REOPEN/MOVE}] 2024-12-07T15:14:06,913 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b393b7989f53c0069cd9db33cd3a7e39, REOPEN/MOVE 2024-12-07T15:14:06,914 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=b393b7989f53c0069cd9db33cd3a7e39, regionState=CLOSING, regionLocation=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:06,915 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b393b7989f53c0069cd9db33cd3a7e39, REOPEN/MOVE because future has completed 2024-12-07T15:14:06,916 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-07T15:14:06,916 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE, hasLock=false; CloseRegionProcedure b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401}] 2024-12-07T15:14:07,069 INFO [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(122): Close b393b7989f53c0069cd9db33cd3a7e39 2024-12-07T15:14:07,069 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-07T15:14:07,069 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1722): Closing b393b7989f53c0069cd9db33cd3a7e39, disabling compactions & flushes 2024-12-07T15:14:07,069 INFO [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:07,069 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:07,069 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. after waiting 0 ms 2024-12-07T15:14:07,069 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:07,074 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-07T15:14:07,075 INFO [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:07,075 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1676): Region close journal for b393b7989f53c0069cd9db33cd3a7e39: Waiting for close lock at 1733584447069Running coprocessor pre-close hooks at 1733584447069Disabling compacts and flushes for region at 1733584447069Disabling writes for close at 1733584447069Writing region close event to WAL at 1733584447070 (+1 ms)Running coprocessor post-close hooks at 1733584447075 (+5 ms)Closed at 1733584447075 2024-12-07T15:14:07,075 WARN [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegionServer(3299): Not adding moved region record: b393b7989f53c0069cd9db33cd3a7e39 to self. 2024-12-07T15:14:07,077 INFO [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(157): Closed b393b7989f53c0069cd9db33cd3a7e39 2024-12-07T15:14:07,078 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=b393b7989f53c0069cd9db33cd3a7e39, regionState=CLOSED 2024-12-07T15:14:07,081 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=37, state=RUNNABLE, hasLock=false; CloseRegionProcedure b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 because future has completed 2024-12-07T15:14:07,084 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-12-07T15:14:07,084 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; CloseRegionProcedure b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 in 166 msec 2024-12-07T15:14:07,085 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b393b7989f53c0069cd9db33cd3a7e39, REOPEN/MOVE; state=CLOSED, location=d6797ebe9c52,46009,1733584418401; forceNewPlan=false, retain=true 2024-12-07T15:14:07,236 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=b393b7989f53c0069cd9db33cd3a7e39, regionState=OPENING, regionLocation=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:07,239 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b393b7989f53c0069cd9db33cd3a7e39, REOPEN/MOVE because future has completed 2024-12-07T15:14:07,239 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401}] 2024-12-07T15:14:07,395 INFO [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:07,396 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} 2024-12-07T15:14:07,396 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees b393b7989f53c0069cd9db33cd3a7e39 2024-12-07T15:14:07,396 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T15:14:07,396 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for b393b7989f53c0069cd9db33cd3a7e39 2024-12-07T15:14:07,396 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for b393b7989f53c0069cd9db33cd3a7e39 2024-12-07T15:14:07,401 INFO [StoreOpener-b393b7989f53c0069cd9db33cd3a7e39-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region b393b7989f53c0069cd9db33cd3a7e39 2024-12-07T15:14:07,401 INFO [StoreOpener-b393b7989f53c0069cd9db33cd3a7e39-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-07T15:14:07,406 INFO [StoreOpener-b393b7989f53c0069cd9db33cd3a7e39-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b393b7989f53c0069cd9db33cd3a7e39 columnFamilyName A 2024-12-07T15:14:07,408 DEBUG [StoreOpener-b393b7989f53c0069cd9db33cd3a7e39-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:07,408 INFO [StoreOpener-b393b7989f53c0069cd9db33cd3a7e39-1 {}] regionserver.HStore(327): Store=b393b7989f53c0069cd9db33cd3a7e39/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T15:14:07,409 INFO [StoreOpener-b393b7989f53c0069cd9db33cd3a7e39-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region b393b7989f53c0069cd9db33cd3a7e39 2024-12-07T15:14:07,409 INFO [StoreOpener-b393b7989f53c0069cd9db33cd3a7e39-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-07T15:14:07,410 INFO [StoreOpener-b393b7989f53c0069cd9db33cd3a7e39-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b393b7989f53c0069cd9db33cd3a7e39 columnFamilyName B 2024-12-07T15:14:07,410 DEBUG [StoreOpener-b393b7989f53c0069cd9db33cd3a7e39-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:07,410 INFO [StoreOpener-b393b7989f53c0069cd9db33cd3a7e39-1 {}] regionserver.HStore(327): Store=b393b7989f53c0069cd9db33cd3a7e39/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T15:14:07,410 INFO [StoreOpener-b393b7989f53c0069cd9db33cd3a7e39-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region b393b7989f53c0069cd9db33cd3a7e39 2024-12-07T15:14:07,411 INFO [StoreOpener-b393b7989f53c0069cd9db33cd3a7e39-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-07T15:14:07,411 INFO [StoreOpener-b393b7989f53c0069cd9db33cd3a7e39-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b393b7989f53c0069cd9db33cd3a7e39 columnFamilyName C 2024-12-07T15:14:07,411 DEBUG [StoreOpener-b393b7989f53c0069cd9db33cd3a7e39-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:07,411 INFO [StoreOpener-b393b7989f53c0069cd9db33cd3a7e39-1 {}] regionserver.HStore(327): Store=b393b7989f53c0069cd9db33cd3a7e39/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T15:14:07,412 INFO [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:07,412 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for b393b7989f53c0069cd9db33cd3a7e39 2024-12-07T15:14:07,413 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39 2024-12-07T15:14:07,414 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39 2024-12-07T15:14:07,415 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for b393b7989f53c0069cd9db33cd3a7e39 2024-12-07T15:14:07,415 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for b393b7989f53c0069cd9db33cd3a7e39 2024-12-07T15:14:07,416 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-07T15:14:07,417 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for b393b7989f53c0069cd9db33cd3a7e39 2024-12-07T15:14:07,418 INFO [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened b393b7989f53c0069cd9db33cd3a7e39; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70374403, jitterRate=0.04866032302379608}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-07T15:14:07,418 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b393b7989f53c0069cd9db33cd3a7e39 2024-12-07T15:14:07,419 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for b393b7989f53c0069cd9db33cd3a7e39: Running coprocessor pre-open hook at 1733584447397Writing region info on filesystem at 1733584447397Initializing all the Stores at 1733584447397Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733584447397Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733584447397Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733584447397Cleaning up temporary data from old regions at 1733584447415 (+18 ms)Running coprocessor post-open hooks at 1733584447418 (+3 ms)Region opened successfully at 1733584447419 (+1 ms) 2024-12-07T15:14:07,420 INFO [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., pid=39, masterSystemTime=1733584447392 2024-12-07T15:14:07,422 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:07,422 INFO [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:07,423 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=b393b7989f53c0069cd9db33cd3a7e39, regionState=OPEN, openSeqNum=5, regionLocation=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:07,424 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 because future has completed 2024-12-07T15:14:07,427 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=37 2024-12-07T15:14:07,427 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=37, state=SUCCESS, hasLock=false; OpenRegionProcedure b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 in 186 msec 2024-12-07T15:14:07,429 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-12-07T15:14:07,429 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b393b7989f53c0069cd9db33cd3a7e39, REOPEN/MOVE in 515 msec 2024-12-07T15:14:07,432 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-12-07T15:14:07,432 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 525 msec 2024-12-07T15:14:07,436 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 550 msec 2024-12-07T15:14:07,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35 2024-12-07T15:14:07,437 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-07T15:14:07,438 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65b7b5d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:14:07,438 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6797ebe9c52,36915,-1 for getting cluster id 2024-12-07T15:14:07,438 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T15:14:07,439 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f8956274-ed33-4e16-82eb-d5d5911ec893' 2024-12-07T15:14:07,439 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T15:14:07,440 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f8956274-ed33-4e16-82eb-d5d5911ec893" 2024-12-07T15:14:07,440 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a4d5785, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:14:07,440 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6797ebe9c52,36915,-1] 2024-12-07T15:14:07,440 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T15:14:07,440 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:14:07,441 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33066, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T15:14:07,441 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e1745a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:14:07,442 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a1845b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:14:07,442 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6797ebe9c52,36915,-1 for getting cluster id 2024-12-07T15:14:07,442 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T15:14:07,443 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f8956274-ed33-4e16-82eb-d5d5911ec893' 2024-12-07T15:14:07,443 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T15:14:07,443 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f8956274-ed33-4e16-82eb-d5d5911ec893" 2024-12-07T15:14:07,443 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1dbb8645, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:14:07,443 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6797ebe9c52,36915,-1] 2024-12-07T15:14:07,443 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T15:14:07,444 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:14:07,444 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33084, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T15:14:07,445 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7db6ea33, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:14:07,446 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@208b436a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:14:07,446 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6797ebe9c52,36915,-1 for getting cluster id 2024-12-07T15:14:07,446 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T15:14:07,447 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f8956274-ed33-4e16-82eb-d5d5911ec893' 2024-12-07T15:14:07,447 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T15:14:07,447 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f8956274-ed33-4e16-82eb-d5d5911ec893" 2024-12-07T15:14:07,447 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d8bbdef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:14:07,447 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6797ebe9c52,36915,-1] 2024-12-07T15:14:07,447 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T15:14:07,447 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:14:07,448 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33098, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T15:14:07,448 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d0e1f3b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:14:07,449 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@349085e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:14:07,449 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6797ebe9c52,36915,-1 for getting cluster id 2024-12-07T15:14:07,449 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T15:14:07,450 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f8956274-ed33-4e16-82eb-d5d5911ec893' 2024-12-07T15:14:07,450 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T15:14:07,450 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f8956274-ed33-4e16-82eb-d5d5911ec893" 2024-12-07T15:14:07,450 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43a206d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:14:07,450 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6797ebe9c52,36915,-1] 2024-12-07T15:14:07,451 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T15:14:07,451 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:14:07,451 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33108, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T15:14:07,452 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6096dcde, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:14:07,452 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ffb7a39, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:14:07,452 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6797ebe9c52,36915,-1 for getting cluster id 2024-12-07T15:14:07,453 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T15:14:07,453 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f8956274-ed33-4e16-82eb-d5d5911ec893' 2024-12-07T15:14:07,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T15:14:07,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f8956274-ed33-4e16-82eb-d5d5911ec893" 2024-12-07T15:14:07,454 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67355c0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:14:07,454 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6797ebe9c52,36915,-1] 2024-12-07T15:14:07,454 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T15:14:07,454 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:14:07,454 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33128, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T15:14:07,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65c92322, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:14:07,456 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d21c225, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:14:07,456 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6797ebe9c52,36915,-1 for getting cluster id 2024-12-07T15:14:07,456 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T15:14:07,456 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f8956274-ed33-4e16-82eb-d5d5911ec893' 2024-12-07T15:14:07,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T15:14:07,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f8956274-ed33-4e16-82eb-d5d5911ec893" 2024-12-07T15:14:07,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56959aad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:14:07,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6797ebe9c52,36915,-1] 2024-12-07T15:14:07,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T15:14:07,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:14:07,458 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33142, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T15:14:07,458 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@645797d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:14:07,459 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a92e196, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:14:07,459 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6797ebe9c52,36915,-1 for getting cluster id 2024-12-07T15:14:07,459 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T15:14:07,460 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f8956274-ed33-4e16-82eb-d5d5911ec893' 2024-12-07T15:14:07,460 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T15:14:07,460 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f8956274-ed33-4e16-82eb-d5d5911ec893" 2024-12-07T15:14:07,460 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34064a4d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:14:07,460 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6797ebe9c52,36915,-1] 2024-12-07T15:14:07,460 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T15:14:07,460 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:14:07,461 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33150, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T15:14:07,462 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3034141e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:14:07,464 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3806f988, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:14:07,464 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6797ebe9c52,36915,-1 for getting cluster id 2024-12-07T15:14:07,464 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T15:14:07,464 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f8956274-ed33-4e16-82eb-d5d5911ec893' 2024-12-07T15:14:07,464 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T15:14:07,464 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f8956274-ed33-4e16-82eb-d5d5911ec893" 2024-12-07T15:14:07,465 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4938e2ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:14:07,465 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6797ebe9c52,36915,-1] 2024-12-07T15:14:07,465 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T15:14:07,465 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:14:07,465 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33172, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T15:14:07,466 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10a94543, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:14:07,467 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@494eb83f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:14:07,467 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6797ebe9c52,36915,-1 for getting cluster id 2024-12-07T15:14:07,467 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T15:14:07,467 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f8956274-ed33-4e16-82eb-d5d5911ec893' 2024-12-07T15:14:07,468 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T15:14:07,468 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f8956274-ed33-4e16-82eb-d5d5911ec893" 2024-12-07T15:14:07,468 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8dbaf9a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:14:07,468 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6797ebe9c52,36915,-1] 2024-12-07T15:14:07,468 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T15:14:07,468 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:14:07,469 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33192, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T15:14:07,469 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b8288ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:14:07,470 DEBUG [Thread-619 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T15:14:07,470 DEBUG [Thread-623 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T15:14:07,470 DEBUG [Thread-606 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T15:14:07,470 DEBUG [Thread-621 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T15:14:07,471 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6797ebe9c52,46009,1733584418401, seqNum=-1] 2024-12-07T15:14:07,471 DEBUG [Thread-617 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T15:14:07,471 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6797ebe9c52,46009,1733584418401, seqNum=-1] 2024-12-07T15:14:07,471 DEBUG [Thread-623 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6797ebe9c52,46009,1733584418401, seqNum=-1] 2024-12-07T15:14:07,472 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T15:14:07,472 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6797ebe9c52,46009,1733584418401, seqNum=-1] 2024-12-07T15:14:07,472 DEBUG [Thread-614 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T15:14:07,472 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T15:14:07,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6797ebe9c52,46009,1733584418401, seqNum=-1] 2024-12-07T15:14:07,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T15:14:07,473 DEBUG [Thread-612 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T15:14:07,473 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37972, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T15:14:07,473 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T15:14:07,473 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37966, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T15:14:07,474 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6797ebe9c52,46009,1733584418401, seqNum=-1] 2024-12-07T15:14:07,474 DEBUG [Thread-612 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T15:14:07,474 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37990, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T15:14:07,474 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37988, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T15:14:07,475 DEBUG [Thread-623 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T15:14:07,475 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38006, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T15:14:07,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6797ebe9c52,46009,1733584418401, seqNum=-1] 2024-12-07T15:14:07,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T15:14:07,476 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38008, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T15:14:07,476 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38020, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T15:14:07,477 DEBUG [Thread-608 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T15:14:07,477 DEBUG [Thread-610 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T15:14:07,477 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5] 2024-12-07T15:14:07,478 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6797ebe9c52,46009,1733584418401, seqNum=-1] 2024-12-07T15:14:07,478 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T15:14:07,478 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6797ebe9c52,46009,1733584418401, seqNum=-1] 2024-12-07T15:14:07,479 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T15:14:07,479 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5] 2024-12-07T15:14:07,479 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38036, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T15:14:07,480 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T15:14:07,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-12-07T15:14:07,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-07T15:14:07,482 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T15:14:07,483 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T15:14:07,483 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T15:14:07,483 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5] 2024-12-07T15:14:07,483 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5] 2024-12-07T15:14:07,485 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38038, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T15:14:07,485 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5] 2024-12-07T15:14:07,486 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5] 2024-12-07T15:14:07,486 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5] 2024-12-07T15:14:07,487 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5] 2024-12-07T15:14:07,487 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5] 2024-12-07T15:14:07,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on b393b7989f53c0069cd9db33cd3a7e39 2024-12-07T15:14:07,500 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b393b7989f53c0069cd9db33cd3a7e39 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-07T15:14:07,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b393b7989f53c0069cd9db33cd3a7e39, store=A 2024-12-07T15:14:07,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:14:07,506 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b393b7989f53c0069cd9db33cd3a7e39, store=B 2024-12-07T15:14:07,506 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:14:07,506 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b393b7989f53c0069cd9db33cd3a7e39, store=C 2024-12-07T15:14:07,506 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:14:07,567 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733584447545, max timestamp=1733584447559Num uniques 859; 2024-12-07T15:14:07,567 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733584447545, max timestamp=1733584447559Num uniques 859; 2024-12-07T15:14:07,567 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-07T15:14:07,567 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-07T15:14:07,571 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733584447545, max timestamp=1733584447559Num uniques 859; 2024-12-07T15:14:07,571 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-07T15:14:07,580 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207c8d9f5416b0a417e861bc3d42d4b0e18_b393b7989f53c0069cd9db33cd3a7e39 is 50, key is test_row_0/A:col10/1733584447485/Put/seqid=0 2024-12-07T15:14:07,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-07T15:14:07,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741944_1120 (size=12154) 2024-12-07T15:14:07,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:07,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38006 deadline: 1733584507617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:07,634 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:07,634 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:07,634 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:07,640 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:07,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:07,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37972 deadline: 1733584507616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:07,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38020 deadline: 1733584507620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:07,641 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:07,641 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:07,641 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:07,641 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:07,641 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:07,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38036 deadline: 1733584507621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:07,641 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:07,641 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:07,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:07,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:07,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:07,645 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-07T15:14:07,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:07,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. as already flushing 2024-12-07T15:14:07,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:07,646 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:07,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:07,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:07,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:07,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38038 deadline: 1733584507632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:07,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:07,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:07,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:07,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:07,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38006 deadline: 1733584507743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:07,750 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:07,750 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:07,750 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:07,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:07,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37972 deadline: 1733584507754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:07,759 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:07,759 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:07,759 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:07,759 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:07,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38036 deadline: 1733584507754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:07,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:07,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:07,759 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:07,760 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:07,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38020 deadline: 1733584507754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:07,760 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:07,760 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:07,760 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:07,760 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:07,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38038 deadline: 1733584507755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:07,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:07,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:07,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:07,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-07T15:14:07,800 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-07T15:14:07,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:07,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. as already flushing 2024-12-07T15:14:07,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:07,800 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:07,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:07,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:07,955 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-07T15:14:07,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:07,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. as already flushing 2024-12-07T15:14:07,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:07,955 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:07,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:07,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:07,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:07,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38006 deadline: 1733584507954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:07,962 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:07,962 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:07,962 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:07,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:07,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38020 deadline: 1733584507964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:07,969 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:07,969 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:07,969 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:07,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:07,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37972 deadline: 1733584507966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:07,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:07,970 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:07,970 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:07,970 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:07,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38036 deadline: 1733584507966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:07,971 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:07,971 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:07,971 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:07,977 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:07,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38038 deadline: 1733584507973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:07,977 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:07,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:07,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:08,025 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:08,031 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207c8d9f5416b0a417e861bc3d42d4b0e18_b393b7989f53c0069cd9db33cd3a7e39 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207c8d9f5416b0a417e861bc3d42d4b0e18_b393b7989f53c0069cd9db33cd3a7e39 2024-12-07T15:14:08,032 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/A/1971927d23774c0b8118642758dd550d, store: [table=TestAcidGuarantees family=A region=b393b7989f53c0069cd9db33cd3a7e39] 2024-12-07T15:14:08,041 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/A/1971927d23774c0b8118642758dd550d is 175, key is test_row_0/A:col10/1733584447485/Put/seqid=0 2024-12-07T15:14:08,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741945_1121 (size=30955) 2024-12-07T15:14:08,069 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/A/1971927d23774c0b8118642758dd550d 2024-12-07T15:14:08,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-07T15:14:08,109 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-07T15:14:08,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:08,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. as already flushing 2024-12-07T15:14:08,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:08,113 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:08,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:08,113 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/B/8b1f614b9b444a22817914dae302cd4e is 50, key is test_row_0/B:col10/1733584447485/Put/seqid=0 2024-12-07T15:14:08,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:08,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741946_1122 (size=12001) 2024-12-07T15:14:08,149 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/B/8b1f614b9b444a22817914dae302cd4e 2024-12-07T15:14:08,181 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/C/2d9caaf98594437fbddfb9b2815d6a5c is 50, key is test_row_0/C:col10/1733584447485/Put/seqid=0 2024-12-07T15:14:08,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741947_1123 (size=12001) 2024-12-07T15:14:08,226 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/C/2d9caaf98594437fbddfb9b2815d6a5c 2024-12-07T15:14:08,232 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/A/1971927d23774c0b8118642758dd550d as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/1971927d23774c0b8118642758dd550d 2024-12-07T15:14:08,241 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/1971927d23774c0b8118642758dd550d, entries=150, sequenceid=15, filesize=30.2 K 2024-12-07T15:14:08,244 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/B/8b1f614b9b444a22817914dae302cd4e as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/8b1f614b9b444a22817914dae302cd4e 2024-12-07T15:14:08,251 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/8b1f614b9b444a22817914dae302cd4e, entries=150, sequenceid=15, filesize=11.7 K 2024-12-07T15:14:08,252 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/C/2d9caaf98594437fbddfb9b2815d6a5c as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/2d9caaf98594437fbddfb9b2815d6a5c 2024-12-07T15:14:08,259 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/2d9caaf98594437fbddfb9b2815d6a5c, entries=150, sequenceid=15, filesize=11.7 K 2024-12-07T15:14:08,262 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=125.23 KB/128240 for b393b7989f53c0069cd9db33cd3a7e39 in 761ms, sequenceid=15, compaction requested=false 2024-12-07T15:14:08,262 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b393b7989f53c0069cd9db33cd3a7e39: 2024-12-07T15:14:08,269 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-07T15:14:08,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:08,269 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing b393b7989f53c0069cd9db33cd3a7e39 3/3 column families, dataSize=125.23 KB heapSize=355.30 KB 2024-12-07T15:14:08,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b393b7989f53c0069cd9db33cd3a7e39, store=A 2024-12-07T15:14:08,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T15:14:08,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b393b7989f53c0069cd9db33cd3a7e39, store=B 2024-12-07T15:14:08,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T15:14:08,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b393b7989f53c0069cd9db33cd3a7e39, store=C 2024-12-07T15:14:08,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T15:14:08,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on b393b7989f53c0069cd9db33cd3a7e39 2024-12-07T15:14:08,278 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. as already flushing 2024-12-07T15:14:08,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120751f453b424754828802c5148b688d956_b393b7989f53c0069cd9db33cd3a7e39 is 50, key is test_row_0/A:col10/1733584447617/Put/seqid=0 2024-12-07T15:14:08,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:08,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38020 deadline: 1733584508295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:08,303 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:08,303 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:08,303 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:08,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:08,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37972 deadline: 1733584508298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:08,306 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:08,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38036 deadline: 1733584508299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:08,306 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:08,306 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:08,306 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:08,306 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:08,306 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:08,306 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:08,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:08,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38038 deadline: 1733584508299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:08,307 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:08,307 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:08,307 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:08,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:08,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38006 deadline: 1733584508302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:08,308 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:08,308 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:08,308 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:08,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741948_1124 (size=12154) 2024-12-07T15:14:08,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:08,327 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120751f453b424754828802c5148b688d956_b393b7989f53c0069cd9db33cd3a7e39 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120751f453b424754828802c5148b688d956_b393b7989f53c0069cd9db33cd3a7e39 2024-12-07T15:14:08,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/A/9e9a6a252298461990ada271d3d40639, store: [table=TestAcidGuarantees family=A region=b393b7989f53c0069cd9db33cd3a7e39] 2024-12-07T15:14:08,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/A/9e9a6a252298461990ada271d3d40639 is 175, key is test_row_0/A:col10/1733584447617/Put/seqid=0 2024-12-07T15:14:08,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741949_1125 (size=30955) 2024-12-07T15:14:08,341 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/A/9e9a6a252298461990ada271d3d40639 2024-12-07T15:14:08,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/B/4fb8bff65a0941a6a882157b5253326c is 50, key is test_row_0/B:col10/1733584447617/Put/seqid=0 2024-12-07T15:14:08,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741950_1126 (size=12001) 2024-12-07T15:14:08,389 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/B/4fb8bff65a0941a6a882157b5253326c 2024-12-07T15:14:08,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:08,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38020 deadline: 1733584508413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:08,417 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:08,417 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:08,417 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:08,417 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:08,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38036 deadline: 1733584508414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:08,417 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:08,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:08,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:08,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/C/a78ba985d893400085ff7ff7316f8962 is 50, key is test_row_0/C:col10/1733584447617/Put/seqid=0 2024-12-07T15:14:08,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:08,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37972 deadline: 1733584508414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:08,423 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:08,423 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:08,423 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:08,424 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:08,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38038 deadline: 1733584508414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:08,424 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:08,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38006 deadline: 1733584508414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:08,425 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:08,425 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:08,425 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:08,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:08,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:08,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:08,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741951_1127 (size=12001) 2024-12-07T15:14:08,457 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/C/a78ba985d893400085ff7ff7316f8962 2024-12-07T15:14:08,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/A/9e9a6a252298461990ada271d3d40639 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/9e9a6a252298461990ada271d3d40639 2024-12-07T15:14:08,472 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/9e9a6a252298461990ada271d3d40639, entries=150, sequenceid=50, filesize=30.2 K 2024-12-07T15:14:08,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/B/4fb8bff65a0941a6a882157b5253326c as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/4fb8bff65a0941a6a882157b5253326c 2024-12-07T15:14:08,486 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/4fb8bff65a0941a6a882157b5253326c, entries=150, sequenceid=50, filesize=11.7 K 2024-12-07T15:14:08,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/C/a78ba985d893400085ff7ff7316f8962 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/a78ba985d893400085ff7ff7316f8962 2024-12-07T15:14:08,499 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/a78ba985d893400085ff7ff7316f8962, entries=150, sequenceid=50, filesize=11.7 K 2024-12-07T15:14:08,500 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~125.23 KB/128240, heapSize ~355.25 KB/363776, currentSize=53.67 KB/54960 for b393b7989f53c0069cd9db33cd3a7e39 in 231ms, sequenceid=50, compaction requested=false 2024-12-07T15:14:08,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for b393b7989f53c0069cd9db33cd3a7e39: 2024-12-07T15:14:08,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:08,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-12-07T15:14:08,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-12-07T15:14:08,504 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-12-07T15:14:08,504 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0190 sec 2024-12-07T15:14:08,508 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 1.0260 sec 2024-12-07T15:14:08,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-07T15:14:08,614 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T15:14:08,614 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T15:14:08,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-12-07T15:14:08,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-07T15:14:08,616 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T15:14:08,617 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T15:14:08,618 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T15:14:08,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on b393b7989f53c0069cd9db33cd3a7e39 2024-12-07T15:14:08,630 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b393b7989f53c0069cd9db33cd3a7e39 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-07T15:14:08,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b393b7989f53c0069cd9db33cd3a7e39, store=A 2024-12-07T15:14:08,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:14:08,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b393b7989f53c0069cd9db33cd3a7e39, store=B 2024-12-07T15:14:08,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:14:08,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b393b7989f53c0069cd9db33cd3a7e39, store=C 2024-12-07T15:14:08,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:14:08,656 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207498e5deb01564163b09708bdf681f447_b393b7989f53c0069cd9db33cd3a7e39 is 50, key is test_row_0/A:col10/1733584448628/Put/seqid=0 2024-12-07T15:14:08,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741952_1128 (size=12154) 2024-12-07T15:14:08,673 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:08,681 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207498e5deb01564163b09708bdf681f447_b393b7989f53c0069cd9db33cd3a7e39 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207498e5deb01564163b09708bdf681f447_b393b7989f53c0069cd9db33cd3a7e39 2024-12-07T15:14:08,682 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/A/b38b6d7881ed4f6591553ed7d0524177, store: [table=TestAcidGuarantees family=A region=b393b7989f53c0069cd9db33cd3a7e39] 2024-12-07T15:14:08,683 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/A/b38b6d7881ed4f6591553ed7d0524177 is 175, key is test_row_0/A:col10/1733584448628/Put/seqid=0 2024-12-07T15:14:08,696 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:08,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37972 deadline: 1733584508679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:08,700 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:08,700 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:08,700 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:08,702 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:08,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38036 deadline: 1733584508686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:08,703 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:08,703 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:08,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:08,703 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:08,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38038 deadline: 1733584508691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:08,704 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:08,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:08,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:08,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:08,704 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:08,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38020 deadline: 1733584508696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:08,705 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:08,705 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:08,705 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:08,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38006 deadline: 1733584508693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:08,721 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:08,721 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:08,721 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:08,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-07T15:14:08,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741953_1129 (size=30955) 2024-12-07T15:14:08,770 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-07T15:14:08,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:08,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. as already flushing 2024-12-07T15:14:08,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:08,771 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:08,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:08,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:08,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:08,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37972 deadline: 1733584508804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:08,813 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:08,813 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:08,813 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:08,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:08,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38036 deadline: 1733584508813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:08,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:08,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:08,821 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:08,821 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:08,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38038 deadline: 1733584508813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:08,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:08,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:08,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:08,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:08,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38020 deadline: 1733584508814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:08,822 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:08,822 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:08,822 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:08,832 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:08,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38006 deadline: 1733584508825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:08,833 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:08,833 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:08,833 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:08,925 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-07T15:14:08,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:08,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. as already flushing 2024-12-07T15:14:08,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:08,925 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:08,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:08,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:08,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-07T15:14:09,027 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:09,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37972 deadline: 1733584509025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:09,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:09,028 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:09,028 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:09,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38036 deadline: 1733584509025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:09,028 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:09,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:09,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:09,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:09,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:09,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38038 deadline: 1733584509034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:09,037 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:09,038 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:09,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:09,038 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:09,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38020 deadline: 1733584509034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:09,038 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:09,038 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:09,039 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:09,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:09,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38006 deadline: 1733584509043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:09,045 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:09,045 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:09,045 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:09,079 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-07T15:14:09,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:09,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. as already flushing 2024-12-07T15:14:09,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:09,080 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:09,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:09,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:09,156 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=62, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/A/b38b6d7881ed4f6591553ed7d0524177 2024-12-07T15:14:09,171 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/B/26861b39ff434574954ac1130e531dff is 50, key is test_row_0/B:col10/1733584448628/Put/seqid=0 2024-12-07T15:14:09,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741954_1130 (size=12001) 2024-12-07T15:14:09,221 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/B/26861b39ff434574954ac1130e531dff 2024-12-07T15:14:09,234 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-07T15:14:09,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:09,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. as already flushing 2024-12-07T15:14:09,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:09,235 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:09,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:09,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:09,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-07T15:14:09,250 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/C/c86814276f8448b0b60387abe9dd8615 is 50, key is test_row_0/C:col10/1733584448628/Put/seqid=0 2024-12-07T15:14:09,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741955_1131 (size=12001) 2024-12-07T15:14:09,286 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/C/c86814276f8448b0b60387abe9dd8615 2024-12-07T15:14:09,298 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/A/b38b6d7881ed4f6591553ed7d0524177 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/b38b6d7881ed4f6591553ed7d0524177 2024-12-07T15:14:09,306 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/b38b6d7881ed4f6591553ed7d0524177, entries=150, sequenceid=62, filesize=30.2 K 2024-12-07T15:14:09,315 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/B/26861b39ff434574954ac1130e531dff as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/26861b39ff434574954ac1130e531dff 2024-12-07T15:14:09,322 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/26861b39ff434574954ac1130e531dff, entries=150, sequenceid=62, filesize=11.7 K 2024-12-07T15:14:09,324 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/C/c86814276f8448b0b60387abe9dd8615 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/c86814276f8448b0b60387abe9dd8615 2024-12-07T15:14:09,331 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/c86814276f8448b0b60387abe9dd8615, entries=150, sequenceid=62, filesize=11.7 K 2024-12-07T15:14:09,333 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for b393b7989f53c0069cd9db33cd3a7e39 in 703ms, sequenceid=62, compaction requested=true 2024-12-07T15:14:09,333 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b393b7989f53c0069cd9db33cd3a7e39: 2024-12-07T15:14:09,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b393b7989f53c0069cd9db33cd3a7e39:A, priority=-2147483648, current under compaction store size is 1 2024-12-07T15:14:09,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:14:09,333 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:14:09,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b393b7989f53c0069cd9db33cd3a7e39:B, priority=-2147483648, current under compaction store size is 2 2024-12-07T15:14:09,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:14:09,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b393b7989f53c0069cd9db33cd3a7e39:C, priority=-2147483648, current under compaction store size is 3 2024-12-07T15:14:09,333 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:14:09,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T15:14:09,335 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:14:09,336 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:14:09,336 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1541): b393b7989f53c0069cd9db33cd3a7e39/A is initiating minor compaction (all files) 2024-12-07T15:14:09,336 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1541): b393b7989f53c0069cd9db33cd3a7e39/B is initiating minor compaction (all files) 2024-12-07T15:14:09,336 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b393b7989f53c0069cd9db33cd3a7e39/A in TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:09,336 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b393b7989f53c0069cd9db33cd3a7e39/B in TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:09,336 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/8b1f614b9b444a22817914dae302cd4e, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/4fb8bff65a0941a6a882157b5253326c, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/26861b39ff434574954ac1130e531dff] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp, totalSize=35.2 K 2024-12-07T15:14:09,336 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/1971927d23774c0b8118642758dd550d, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/9e9a6a252298461990ada271d3d40639, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/b38b6d7881ed4f6591553ed7d0524177] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp, totalSize=90.7 K 2024-12-07T15:14:09,336 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:09,336 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. files: [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/1971927d23774c0b8118642758dd550d, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/9e9a6a252298461990ada271d3d40639, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/b38b6d7881ed4f6591553ed7d0524177] 2024-12-07T15:14:09,337 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b1f614b9b444a22817914dae302cd4e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733584447485 2024-12-07T15:14:09,337 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 4fb8bff65a0941a6a882157b5253326c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733584447616 2024-12-07T15:14:09,338 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1971927d23774c0b8118642758dd550d, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733584447485 2024-12-07T15:14:09,342 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 26861b39ff434574954ac1130e531dff, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733584448297 2024-12-07T15:14:09,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on b393b7989f53c0069cd9db33cd3a7e39 2024-12-07T15:14:09,342 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b393b7989f53c0069cd9db33cd3a7e39 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-07T15:14:09,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b393b7989f53c0069cd9db33cd3a7e39, store=A 2024-12-07T15:14:09,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T15:14:09,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b393b7989f53c0069cd9db33cd3a7e39, store=B 2024-12-07T15:14:09,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T15:14:09,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b393b7989f53c0069cd9db33cd3a7e39, store=C 2024-12-07T15:14:09,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T15:14:09,347 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e9a6a252298461990ada271d3d40639, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733584447616 2024-12-07T15:14:09,349 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting b38b6d7881ed4f6591553ed7d0524177, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733584448297 2024-12-07T15:14:09,367 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412070a1fab824a144234aabf25850171fdea_b393b7989f53c0069cd9db33cd3a7e39 is 50, key is test_row_0/A:col10/1733584448683/Put/seqid=0 2024-12-07T15:14:09,371 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b393b7989f53c0069cd9db33cd3a7e39#B#compaction#114 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:14:09,371 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/B/929db56ce8cc4600933bc5209587d6f6 is 50, key is test_row_0/B:col10/1733584448628/Put/seqid=0 2024-12-07T15:14:09,374 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b393b7989f53c0069cd9db33cd3a7e39] 2024-12-07T15:14:09,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:09,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38020 deadline: 1733584509374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:09,389 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:09,389 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:09,389 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:09,391 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-07T15:14:09,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:09,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. as already flushing 2024-12-07T15:14:09,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:09,391 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:09,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:09,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:09,397 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241207aaf14ac16e274986b5df191d393889c2_b393b7989f53c0069cd9db33cd3a7e39 store=[table=TestAcidGuarantees family=A region=b393b7989f53c0069cd9db33cd3a7e39] 2024-12-07T15:14:09,398 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:09,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38036 deadline: 1733584509378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:09,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:09,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:09,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:09,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:09,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38006 deadline: 1733584509378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:09,399 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:09,399 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:09,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:09,400 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:09,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38038 deadline: 1733584509379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:09,402 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:09,402 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:09,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37972 deadline: 1733584509391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:09,402 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:09,402 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:09,402 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:09,402 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:09,402 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:09,403 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241207aaf14ac16e274986b5df191d393889c2_b393b7989f53c0069cd9db33cd3a7e39, store=[table=TestAcidGuarantees family=A region=b393b7989f53c0069cd9db33cd3a7e39] 2024-12-07T15:14:09,403 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207aaf14ac16e274986b5df191d393889c2_b393b7989f53c0069cd9db33cd3a7e39 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b393b7989f53c0069cd9db33cd3a7e39] 2024-12-07T15:14:09,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741957_1133 (size=12104) 2024-12-07T15:14:09,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741956_1132 (size=14594) 2024-12-07T15:14:09,423 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:09,429 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412070a1fab824a144234aabf25850171fdea_b393b7989f53c0069cd9db33cd3a7e39 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412070a1fab824a144234aabf25850171fdea_b393b7989f53c0069cd9db33cd3a7e39 2024-12-07T15:14:09,430 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/B/929db56ce8cc4600933bc5209587d6f6 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/929db56ce8cc4600933bc5209587d6f6 2024-12-07T15:14:09,431 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/A/86dac675f9954267b100910c6d57cda6, store: [table=TestAcidGuarantees family=A region=b393b7989f53c0069cd9db33cd3a7e39] 2024-12-07T15:14:09,432 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/A/86dac675f9954267b100910c6d57cda6 is 175, key is test_row_0/A:col10/1733584448683/Put/seqid=0 2024-12-07T15:14:09,440 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b393b7989f53c0069cd9db33cd3a7e39/B of b393b7989f53c0069cd9db33cd3a7e39 into 929db56ce8cc4600933bc5209587d6f6(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:14:09,440 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b393b7989f53c0069cd9db33cd3a7e39: 2024-12-07T15:14:09,440 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., storeName=b393b7989f53c0069cd9db33cd3a7e39/B, priority=13, startTime=1733584449333; duration=0sec 2024-12-07T15:14:09,440 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T15:14:09,440 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b393b7989f53c0069cd9db33cd3a7e39:B 2024-12-07T15:14:09,440 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:14:09,441 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:14:09,441 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1541): b393b7989f53c0069cd9db33cd3a7e39/C is initiating minor compaction (all files) 2024-12-07T15:14:09,442 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b393b7989f53c0069cd9db33cd3a7e39/C in TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:09,442 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/2d9caaf98594437fbddfb9b2815d6a5c, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/a78ba985d893400085ff7ff7316f8962, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/c86814276f8448b0b60387abe9dd8615] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp, totalSize=35.2 K 2024-12-07T15:14:09,442 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d9caaf98594437fbddfb9b2815d6a5c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733584447485 2024-12-07T15:14:09,443 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting a78ba985d893400085ff7ff7316f8962, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733584447616 2024-12-07T15:14:09,443 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting c86814276f8448b0b60387abe9dd8615, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733584448297 2024-12-07T15:14:09,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741958_1134 (size=4469) 2024-12-07T15:14:09,454 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b393b7989f53c0069cd9db33cd3a7e39#A#compaction#115 average throughput is 0.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:14:09,455 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/A/808e0e4e7dbe4b558f1ea1218a87e52b is 175, key is test_row_0/A:col10/1733584448628/Put/seqid=0 2024-12-07T15:14:09,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741959_1135 (size=39549) 2024-12-07T15:14:09,494 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=83, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/A/86dac675f9954267b100910c6d57cda6 2024-12-07T15:14:09,498 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:09,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38020 deadline: 1733584509493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:09,498 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:09,498 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:09,498 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:09,499 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b393b7989f53c0069cd9db33cd3a7e39#C#compaction#116 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:14:09,499 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/C/e5ce57d984614cf993f7855d8222857a is 50, key is test_row_0/C:col10/1733584448628/Put/seqid=0 2024-12-07T15:14:09,513 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:09,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38036 deadline: 1733584509504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:09,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:09,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38006 deadline: 1733584509504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:09,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:09,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:09,515 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:09,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:09,515 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:09,515 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:09,518 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:09,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37972 deadline: 1733584509514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:09,519 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:09,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38038 deadline: 1733584509514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:09,519 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:09,519 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:09,519 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:09,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:09,520 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:09,520 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:09,524 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/B/7d23a3261a3d4d149c89d6b1a002209a is 50, key is test_row_0/B:col10/1733584448683/Put/seqid=0 2024-12-07T15:14:09,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741960_1136 (size=31058) 2024-12-07T15:14:09,543 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/A/808e0e4e7dbe4b558f1ea1218a87e52b as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/808e0e4e7dbe4b558f1ea1218a87e52b 2024-12-07T15:14:09,545 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-07T15:14:09,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:09,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. as already flushing 2024-12-07T15:14:09,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:09,548 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:09,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:09,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:09,556 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b393b7989f53c0069cd9db33cd3a7e39/A of b393b7989f53c0069cd9db33cd3a7e39 into 808e0e4e7dbe4b558f1ea1218a87e52b(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:14:09,556 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b393b7989f53c0069cd9db33cd3a7e39: 2024-12-07T15:14:09,556 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., storeName=b393b7989f53c0069cd9db33cd3a7e39/A, priority=13, startTime=1733584449333; duration=0sec 2024-12-07T15:14:09,557 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:14:09,557 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b393b7989f53c0069cd9db33cd3a7e39:A 2024-12-07T15:14:09,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741961_1137 (size=12104) 2024-12-07T15:14:09,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741962_1138 (size=12001) 2024-12-07T15:14:09,568 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/B/7d23a3261a3d4d149c89d6b1a002209a 2024-12-07T15:14:09,580 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/C/e5ce57d984614cf993f7855d8222857a as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/e5ce57d984614cf993f7855d8222857a 2024-12-07T15:14:09,591 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/C/710867729db542fa81939e889c8c6f96 is 50, key is test_row_0/C:col10/1733584448683/Put/seqid=0 2024-12-07T15:14:09,596 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b393b7989f53c0069cd9db33cd3a7e39/C of b393b7989f53c0069cd9db33cd3a7e39 into e5ce57d984614cf993f7855d8222857a(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:14:09,596 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b393b7989f53c0069cd9db33cd3a7e39: 2024-12-07T15:14:09,596 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., storeName=b393b7989f53c0069cd9db33cd3a7e39/C, priority=13, startTime=1733584449333; duration=0sec 2024-12-07T15:14:09,597 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:14:09,597 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b393b7989f53c0069cd9db33cd3a7e39:C 2024-12-07T15:14:09,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741963_1139 (size=12001) 2024-12-07T15:14:09,619 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/C/710867729db542fa81939e889c8c6f96 2024-12-07T15:14:09,627 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/A/86dac675f9954267b100910c6d57cda6 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/86dac675f9954267b100910c6d57cda6 2024-12-07T15:14:09,635 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/86dac675f9954267b100910c6d57cda6, entries=200, sequenceid=83, filesize=38.6 K 2024-12-07T15:14:09,637 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/B/7d23a3261a3d4d149c89d6b1a002209a as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/7d23a3261a3d4d149c89d6b1a002209a 2024-12-07T15:14:09,644 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/7d23a3261a3d4d149c89d6b1a002209a, entries=150, sequenceid=83, filesize=11.7 K 2024-12-07T15:14:09,647 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/C/710867729db542fa81939e889c8c6f96 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/710867729db542fa81939e889c8c6f96 2024-12-07T15:14:09,694 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/710867729db542fa81939e889c8c6f96, entries=150, sequenceid=83, filesize=11.7 K 2024-12-07T15:14:09,702 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-07T15:14:09,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:09,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. as already flushing 2024-12-07T15:14:09,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:09,703 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:09,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:09,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:09,706 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for b393b7989f53c0069cd9db33cd3a7e39 in 364ms, sequenceid=83, compaction requested=false 2024-12-07T15:14:09,707 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b393b7989f53c0069cd9db33cd3a7e39: 2024-12-07T15:14:09,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on b393b7989f53c0069cd9db33cd3a7e39 2024-12-07T15:14:09,712 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b393b7989f53c0069cd9db33cd3a7e39 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-07T15:14:09,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b393b7989f53c0069cd9db33cd3a7e39, store=A 2024-12-07T15:14:09,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:14:09,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b393b7989f53c0069cd9db33cd3a7e39, store=B 2024-12-07T15:14:09,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:14:09,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b393b7989f53c0069cd9db33cd3a7e39, store=C 2024-12-07T15:14:09,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:14:09,742 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207c72a2af2820d48b7bd49ea7a036aa950_b393b7989f53c0069cd9db33cd3a7e39 is 50, key is test_row_0/A:col10/1733584449377/Put/seqid=0 2024-12-07T15:14:09,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-07T15:14:09,768 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14144, totalHeapSize=32376, min timestamp=1733584449727, max timestamp=1733584449764Num uniques 859; 2024-12-07T15:14:09,768 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14144, totalHeapSize=32376, min timestamp=1733584449727, max timestamp=1733584449764Num uniques 859; 2024-12-07T15:14:09,768 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=14144, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-07T15:14:09,768 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=14144, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-07T15:14:09,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741964_1140 (size=12154) 2024-12-07T15:14:09,783 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:09,790 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207c72a2af2820d48b7bd49ea7a036aa950_b393b7989f53c0069cd9db33cd3a7e39 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207c72a2af2820d48b7bd49ea7a036aa950_b393b7989f53c0069cd9db33cd3a7e39 2024-12-07T15:14:09,791 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/A/963f1c81ec604ccdbf35aa50cf584592, store: [table=TestAcidGuarantees family=A region=b393b7989f53c0069cd9db33cd3a7e39] 2024-12-07T15:14:09,792 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/A/963f1c81ec604ccdbf35aa50cf584592 is 175, key is test_row_0/A:col10/1733584449377/Put/seqid=0 2024-12-07T15:14:09,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:09,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37972 deadline: 1733584509778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:09,795 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:09,795 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:09,795 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:09,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:09,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38020 deadline: 1733584509781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:09,798 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:09,798 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:09,798 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:09,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:09,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38036 deadline: 1733584509781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:09,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:09,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:09,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:09,802 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:09,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38038 deadline: 1733584509793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:09,802 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:09,802 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:09,802 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:09,802 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:09,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38006 deadline: 1733584509795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:09,803 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:09,803 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:09,803 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:09,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741965_1141 (size=30955) 2024-12-07T15:14:09,819 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=98, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/A/963f1c81ec604ccdbf35aa50cf584592 2024-12-07T15:14:09,830 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/B/b8d5848f24444b32b6c89250a5c75274 is 50, key is test_row_0/B:col10/1733584449377/Put/seqid=0 2024-12-07T15:14:09,857 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-07T15:14:09,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:09,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. as already flushing 2024-12-07T15:14:09,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:09,858 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:09,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:09,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:09,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741966_1142 (size=12001) 2024-12-07T15:14:09,898 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-07T15:14:09,921 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:09,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37972 deadline: 1733584509909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:09,922 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:09,922 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:09,922 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:09,922 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:09,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38036 deadline: 1733584509909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:09,923 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:09,923 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:09,923 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:09,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:09,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38020 deadline: 1733584509909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:09,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:09,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:09,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:09,925 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:09,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38038 deadline: 1733584509914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:09,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:09,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:09,925 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:09,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:09,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38006 deadline: 1733584509914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:09,926 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:09,926 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:09,926 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:10,015 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-07T15:14:10,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:10,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. as already flushing 2024-12-07T15:14:10,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:10,016 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:10,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:10,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:10,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:10,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37972 deadline: 1733584510126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:10,128 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:10,128 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:10,128 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:10,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:10,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38006 deadline: 1733584510134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:10,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:10,139 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:10,139 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:10,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38036 deadline: 1733584510134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:10,139 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:10,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:10,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:10,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:10,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:10,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38020 deadline: 1733584510134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:10,140 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:10,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:10,140 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:10,140 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:10,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38038 deadline: 1733584510134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:10,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:10,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:10,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:10,169 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-07T15:14:10,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:10,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. as already flushing 2024-12-07T15:14:10,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:10,170 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:10,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:10,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:10,280 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/B/b8d5848f24444b32b6c89250a5c75274 2024-12-07T15:14:10,293 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/C/ea8b8007c11e4c6c991d1bfc22df3f3e is 50, key is test_row_0/C:col10/1733584449377/Put/seqid=0 2024-12-07T15:14:10,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741967_1143 (size=12001) 2024-12-07T15:14:10,303 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/C/ea8b8007c11e4c6c991d1bfc22df3f3e 2024-12-07T15:14:10,314 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/A/963f1c81ec604ccdbf35aa50cf584592 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/963f1c81ec604ccdbf35aa50cf584592 2024-12-07T15:14:10,320 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/963f1c81ec604ccdbf35aa50cf584592, entries=150, sequenceid=98, filesize=30.2 K 2024-12-07T15:14:10,321 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/B/b8d5848f24444b32b6c89250a5c75274 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/b8d5848f24444b32b6c89250a5c75274 2024-12-07T15:14:10,323 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-07T15:14:10,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:10,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. as already flushing 2024-12-07T15:14:10,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:10,324 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:10,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:10,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:10,327 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/b8d5848f24444b32b6c89250a5c75274, entries=150, sequenceid=98, filesize=11.7 K 2024-12-07T15:14:10,328 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/C/ea8b8007c11e4c6c991d1bfc22df3f3e as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/ea8b8007c11e4c6c991d1bfc22df3f3e 2024-12-07T15:14:10,333 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/ea8b8007c11e4c6c991d1bfc22df3f3e, entries=150, sequenceid=98, filesize=11.7 K 2024-12-07T15:14:10,334 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=118.53 KB/121370 for b393b7989f53c0069cd9db33cd3a7e39 in 622ms, sequenceid=98, compaction requested=true 2024-12-07T15:14:10,334 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b393b7989f53c0069cd9db33cd3a7e39: 2024-12-07T15:14:10,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b393b7989f53c0069cd9db33cd3a7e39:A, priority=-2147483648, current under compaction store size is 1 2024-12-07T15:14:10,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:14:10,335 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:14:10,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b393b7989f53c0069cd9db33cd3a7e39:B, priority=-2147483648, current under compaction store size is 2 2024-12-07T15:14:10,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:14:10,335 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:14:10,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b393b7989f53c0069cd9db33cd3a7e39:C, priority=-2147483648, current under compaction store size is 3 2024-12-07T15:14:10,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T15:14:10,336 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:14:10,336 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:14:10,336 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1541): b393b7989f53c0069cd9db33cd3a7e39/A is initiating minor compaction (all files) 2024-12-07T15:14:10,336 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1541): b393b7989f53c0069cd9db33cd3a7e39/B is initiating minor compaction (all files) 2024-12-07T15:14:10,337 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b393b7989f53c0069cd9db33cd3a7e39/B in TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:10,337 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b393b7989f53c0069cd9db33cd3a7e39/A in TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:10,338 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/929db56ce8cc4600933bc5209587d6f6, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/7d23a3261a3d4d149c89d6b1a002209a, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/b8d5848f24444b32b6c89250a5c75274] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp, totalSize=35.3 K 2024-12-07T15:14:10,338 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/808e0e4e7dbe4b558f1ea1218a87e52b, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/86dac675f9954267b100910c6d57cda6, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/963f1c81ec604ccdbf35aa50cf584592] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp, totalSize=99.2 K 2024-12-07T15:14:10,338 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:10,338 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. files: [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/808e0e4e7dbe4b558f1ea1218a87e52b, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/86dac675f9954267b100910c6d57cda6, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/963f1c81ec604ccdbf35aa50cf584592] 2024-12-07T15:14:10,338 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 929db56ce8cc4600933bc5209587d6f6, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733584448297 2024-12-07T15:14:10,338 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 808e0e4e7dbe4b558f1ea1218a87e52b, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733584448297 2024-12-07T15:14:10,338 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d23a3261a3d4d149c89d6b1a002209a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1733584448672 2024-12-07T15:14:10,339 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting b8d5848f24444b32b6c89250a5c75274, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733584449377 2024-12-07T15:14:10,339 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 86dac675f9954267b100910c6d57cda6, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1733584448672 2024-12-07T15:14:10,339 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 963f1c81ec604ccdbf35aa50cf584592, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733584449377 2024-12-07T15:14:10,352 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b393b7989f53c0069cd9db33cd3a7e39#B#compaction#122 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:14:10,352 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/B/552958a789f94d69830a836c4e84f4bd is 50, key is test_row_0/B:col10/1733584449377/Put/seqid=0 2024-12-07T15:14:10,357 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b393b7989f53c0069cd9db33cd3a7e39] 2024-12-07T15:14:10,364 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241207b20d492cf6c140d8b41f462d2bc27e44_b393b7989f53c0069cd9db33cd3a7e39 store=[table=TestAcidGuarantees family=A region=b393b7989f53c0069cd9db33cd3a7e39] 2024-12-07T15:14:10,368 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241207b20d492cf6c140d8b41f462d2bc27e44_b393b7989f53c0069cd9db33cd3a7e39, store=[table=TestAcidGuarantees family=A region=b393b7989f53c0069cd9db33cd3a7e39] 2024-12-07T15:14:10,368 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207b20d492cf6c140d8b41f462d2bc27e44_b393b7989f53c0069cd9db33cd3a7e39 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b393b7989f53c0069cd9db33cd3a7e39] 2024-12-07T15:14:10,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741968_1144 (size=12207) 2024-12-07T15:14:10,379 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/B/552958a789f94d69830a836c4e84f4bd as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/552958a789f94d69830a836c4e84f4bd 2024-12-07T15:14:10,388 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b393b7989f53c0069cd9db33cd3a7e39/B of b393b7989f53c0069cd9db33cd3a7e39 into 552958a789f94d69830a836c4e84f4bd(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:14:10,388 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b393b7989f53c0069cd9db33cd3a7e39: 2024-12-07T15:14:10,388 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., storeName=b393b7989f53c0069cd9db33cd3a7e39/B, priority=13, startTime=1733584450335; duration=0sec 2024-12-07T15:14:10,388 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T15:14:10,388 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b393b7989f53c0069cd9db33cd3a7e39:B 2024-12-07T15:14:10,388 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:14:10,391 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:14:10,391 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1541): b393b7989f53c0069cd9db33cd3a7e39/C is initiating minor compaction (all files) 2024-12-07T15:14:10,391 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b393b7989f53c0069cd9db33cd3a7e39/C in TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:10,391 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/e5ce57d984614cf993f7855d8222857a, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/710867729db542fa81939e889c8c6f96, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/ea8b8007c11e4c6c991d1bfc22df3f3e] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp, totalSize=35.3 K 2024-12-07T15:14:10,391 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting e5ce57d984614cf993f7855d8222857a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733584448297 2024-12-07T15:14:10,392 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 710867729db542fa81939e889c8c6f96, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1733584448672 2024-12-07T15:14:10,392 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting ea8b8007c11e4c6c991d1bfc22df3f3e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733584449377 2024-12-07T15:14:10,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741969_1145 (size=4469) 2024-12-07T15:14:10,401 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b393b7989f53c0069cd9db33cd3a7e39#C#compaction#124 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:14:10,402 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/C/a247c9568ce542f2a817d6e46be55dc1 is 50, key is test_row_0/C:col10/1733584449377/Put/seqid=0 2024-12-07T15:14:10,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741970_1146 (size=12207) 2024-12-07T15:14:10,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on b393b7989f53c0069cd9db33cd3a7e39 2024-12-07T15:14:10,438 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b393b7989f53c0069cd9db33cd3a7e39 3/3 column families, dataSize=125.23 KB heapSize=354.02 KB 2024-12-07T15:14:10,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b393b7989f53c0069cd9db33cd3a7e39, store=A 2024-12-07T15:14:10,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T15:14:10,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b393b7989f53c0069cd9db33cd3a7e39, store=B 2024-12-07T15:14:10,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T15:14:10,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b393b7989f53c0069cd9db33cd3a7e39, store=C 2024-12-07T15:14:10,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T15:14:10,453 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412076a9c7617304d471282afacafc39897a5_b393b7989f53c0069cd9db33cd3a7e39 is 50, key is test_row_0/A:col10/1733584449792/Put/seqid=0 2024-12-07T15:14:10,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:10,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38038 deadline: 1733584510451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:10,460 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:10,460 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:10,460 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:10,460 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:10,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38036 deadline: 1733584510454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:10,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:10,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:10,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:10,461 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:10,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38020 deadline: 1733584510458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:10,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:10,462 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:10,462 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:10,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38006 deadline: 1733584510458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:10,462 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:10,462 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:10,462 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:10,462 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:10,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741971_1147 (size=14594) 2024-12-07T15:14:10,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:10,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37972 deadline: 1733584510461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:10,471 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:10,472 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:10,472 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:10,472 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:10,477 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-07T15:14:10,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:10,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. as already flushing 2024-12-07T15:14:10,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:10,477 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:10,477 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412076a9c7617304d471282afacafc39897a5_b393b7989f53c0069cd9db33cd3a7e39 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412076a9c7617304d471282afacafc39897a5_b393b7989f53c0069cd9db33cd3a7e39 2024-12-07T15:14:10,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:10,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:10,480 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/A/c5a156582a3446ad88d8c14897191d62, store: [table=TestAcidGuarantees family=A region=b393b7989f53c0069cd9db33cd3a7e39] 2024-12-07T15:14:10,481 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/A/c5a156582a3446ad88d8c14897191d62 is 175, key is test_row_0/A:col10/1733584449792/Put/seqid=0 2024-12-07T15:14:10,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741972_1148 (size=39549) 2024-12-07T15:14:10,565 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:10,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38038 deadline: 1733584510564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:10,566 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:10,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:10,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38036 deadline: 1733584510564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:10,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:10,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:10,567 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:10,567 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:10,567 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:10,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:10,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38020 deadline: 1733584510573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:10,577 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:10,577 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:10,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:10,577 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:10,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38006 deadline: 1733584510573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:10,578 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:10,578 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:10,578 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:10,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:10,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37972 deadline: 1733584510574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:10,584 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:10,584 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:10,584 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:10,630 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-07T15:14:10,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:10,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. as already flushing 2024-12-07T15:14:10,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:10,631 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:10,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:10,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:10,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-07T15:14:10,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:10,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38038 deadline: 1733584510774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:10,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:10,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:10,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:10,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:10,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38036 deadline: 1733584510774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:10,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:10,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:10,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:10,784 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-07T15:14:10,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:10,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. as already flushing 2024-12-07T15:14:10,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:10,785 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:10,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:10,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:10,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:10,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38006 deadline: 1733584510783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:10,788 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:10,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:10,788 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:10,788 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:10,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38020 deadline: 1733584510784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:10,788 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:10,788 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:10,788 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:10,795 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b393b7989f53c0069cd9db33cd3a7e39#A#compaction#123 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:14:10,796 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/A/c1a15f2e0802400c99c942950da40098 is 175, key is test_row_0/A:col10/1733584449377/Put/seqid=0 2024-12-07T15:14:10,800 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:10,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37972 deadline: 1733584510794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:10,801 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:10,801 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:10,801 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:10,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741973_1149 (size=31161) 2024-12-07T15:14:10,811 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/A/c1a15f2e0802400c99c942950da40098 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/c1a15f2e0802400c99c942950da40098 2024-12-07T15:14:10,822 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/C/a247c9568ce542f2a817d6e46be55dc1 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/a247c9568ce542f2a817d6e46be55dc1 2024-12-07T15:14:10,824 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b393b7989f53c0069cd9db33cd3a7e39/A of b393b7989f53c0069cd9db33cd3a7e39 into c1a15f2e0802400c99c942950da40098(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:14:10,824 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b393b7989f53c0069cd9db33cd3a7e39: 2024-12-07T15:14:10,824 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., storeName=b393b7989f53c0069cd9db33cd3a7e39/A, priority=13, startTime=1733584450334; duration=0sec 2024-12-07T15:14:10,824 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:14:10,824 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b393b7989f53c0069cd9db33cd3a7e39:A 2024-12-07T15:14:10,836 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b393b7989f53c0069cd9db33cd3a7e39/C of b393b7989f53c0069cd9db33cd3a7e39 into a247c9568ce542f2a817d6e46be55dc1(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:14:10,836 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b393b7989f53c0069cd9db33cd3a7e39: 2024-12-07T15:14:10,836 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., storeName=b393b7989f53c0069cd9db33cd3a7e39/C, priority=13, startTime=1733584450335; duration=0sec 2024-12-07T15:14:10,837 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:14:10,837 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b393b7989f53c0069cd9db33cd3a7e39:C 2024-12-07T15:14:10,890 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=128, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/A/c5a156582a3446ad88d8c14897191d62 2024-12-07T15:14:10,901 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/B/92eaac8ad8174f4a9fec575d3136cbc1 is 50, key is test_row_0/B:col10/1733584449792/Put/seqid=0 2024-12-07T15:14:10,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741974_1150 (size=12001) 2024-12-07T15:14:10,908 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/B/92eaac8ad8174f4a9fec575d3136cbc1 2024-12-07T15:14:10,917 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/C/9141de3767344ab09d9b1c671525f76a is 50, key is test_row_0/C:col10/1733584449792/Put/seqid=0 2024-12-07T15:14:10,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741975_1151 (size=12001) 2024-12-07T15:14:10,938 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-07T15:14:10,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:10,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. as already flushing 2024-12-07T15:14:10,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:10,938 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:10,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:10,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:11,084 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:11,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38038 deadline: 1733584511084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:11,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:11,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38036 deadline: 1733584511084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:11,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:11,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:11,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:11,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:11,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:11,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:11,091 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-07T15:14:11,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:11,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. as already flushing 2024-12-07T15:14:11,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:11,092 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:11,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:11,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:11,094 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:11,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38006 deadline: 1733584511093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:11,094 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:11,094 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:11,094 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:11,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:11,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38020 deadline: 1733584511093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:11,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:11,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:11,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:11,116 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:11,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37972 deadline: 1733584511113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:11,117 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:11,117 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:11,117 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:11,245 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-07T15:14:11,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:11,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. as already flushing 2024-12-07T15:14:11,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:11,246 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:11,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:11,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:11,323 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/C/9141de3767344ab09d9b1c671525f76a 2024-12-07T15:14:11,330 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/A/c5a156582a3446ad88d8c14897191d62 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/c5a156582a3446ad88d8c14897191d62 2024-12-07T15:14:11,336 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/c5a156582a3446ad88d8c14897191d62, entries=200, sequenceid=128, filesize=38.6 K 2024-12-07T15:14:11,337 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/B/92eaac8ad8174f4a9fec575d3136cbc1 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/92eaac8ad8174f4a9fec575d3136cbc1 2024-12-07T15:14:11,343 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/92eaac8ad8174f4a9fec575d3136cbc1, entries=150, sequenceid=128, filesize=11.7 K 2024-12-07T15:14:11,345 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/C/9141de3767344ab09d9b1c671525f76a as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/9141de3767344ab09d9b1c671525f76a 2024-12-07T15:14:11,351 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/9141de3767344ab09d9b1c671525f76a, entries=150, sequenceid=128, filesize=11.7 K 2024-12-07T15:14:11,352 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~125.23 KB/128240, heapSize ~353.97 KB/362464, currentSize=53.67 KB/54960 for b393b7989f53c0069cd9db33cd3a7e39 in 914ms, sequenceid=128, compaction requested=false 2024-12-07T15:14:11,353 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b393b7989f53c0069cd9db33cd3a7e39: 2024-12-07T15:14:11,400 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-07T15:14:11,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:11,400 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing b393b7989f53c0069cd9db33cd3a7e39 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-07T15:14:11,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b393b7989f53c0069cd9db33cd3a7e39, store=A 2024-12-07T15:14:11,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:14:11,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b393b7989f53c0069cd9db33cd3a7e39, store=B 2024-12-07T15:14:11,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:14:11,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b393b7989f53c0069cd9db33cd3a7e39, store=C 2024-12-07T15:14:11,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:14:11,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207189346f928b847d780fa433a5c15855f_b393b7989f53c0069cd9db33cd3a7e39 is 50, key is test_row_0/A:col10/1733584450448/Put/seqid=0 2024-12-07T15:14:11,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741976_1152 (size=12304) 2024-12-07T15:14:11,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on b393b7989f53c0069cd9db33cd3a7e39 2024-12-07T15:14:11,595 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. as already flushing 2024-12-07T15:14:11,634 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1733584451622, max timestamp=1733584451631Num uniques 859; 2024-12-07T15:14:11,634 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-07T15:14:11,635 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1733584451622, max timestamp=1733584451631Num uniques 859; 2024-12-07T15:14:11,635 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-07T15:14:11,637 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733584451622, max timestamp=1733584451631Num uniques 859; 2024-12-07T15:14:11,637 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-07T15:14:11,672 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:11,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38036 deadline: 1733584511663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:11,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:11,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:11,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:11,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:11,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38006 deadline: 1733584511664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:11,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:11,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38038 deadline: 1733584511665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:11,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:11,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37972 deadline: 1733584511671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:11,679 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:11,679 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:11,679 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:11,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:11,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:11,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:11,680 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:11,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:11,681 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:11,681 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:11,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38020 deadline: 1733584511673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:11,681 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:11,681 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:11,681 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:11,787 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:11,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:11,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38036 deadline: 1733584511783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:11,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37972 deadline: 1733584511783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:11,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:11,788 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:11,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38020 deadline: 1733584511784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:11,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38038 deadline: 1733584511784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:11,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:11,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:11,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:11,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:11,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:11,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:11,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38006 deadline: 1733584511784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:11,789 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:11,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:11,789 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:11,789 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:11,789 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:11,790 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:11,790 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:11,790 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:11,790 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:11,790 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:11,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:11,834 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207189346f928b847d780fa433a5c15855f_b393b7989f53c0069cd9db33cd3a7e39 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207189346f928b847d780fa433a5c15855f_b393b7989f53c0069cd9db33cd3a7e39 2024-12-07T15:14:11,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/A/a84ed360131f424999ab322f82d81d7e, store: [table=TestAcidGuarantees family=A region=b393b7989f53c0069cd9db33cd3a7e39] 2024-12-07T15:14:11,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/A/a84ed360131f424999ab322f82d81d7e is 175, key is test_row_0/A:col10/1733584450448/Put/seqid=0 2024-12-07T15:14:11,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741977_1153 (size=31105) 2024-12-07T15:14:11,996 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:11,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38036 deadline: 1733584511994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:11,996 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:11,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:11,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:11,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38020 deadline: 1733584511994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:11,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:11,997 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:11,997 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:11,997 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:12,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:12,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37972 deadline: 1733584511994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:12,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:12,001 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:12,002 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:12,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38006 deadline: 1733584511994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:12,002 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:12,002 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:12,002 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:12,002 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:12,002 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:12,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38038 deadline: 1733584511994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:12,002 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:12,002 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:12,002 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:12,241 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=141, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/A/a84ed360131f424999ab322f82d81d7e 2024-12-07T15:14:12,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/B/b030904bef4745bf93ddae147deb18da is 50, key is test_row_0/B:col10/1733584450448/Put/seqid=0 2024-12-07T15:14:12,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741978_1154 (size=12151) 2024-12-07T15:14:12,304 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:12,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38036 deadline: 1733584512303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:12,304 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:12,305 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:12,305 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:12,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38020 deadline: 1733584512303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:12,305 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:12,305 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:12,305 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:12,305 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:12,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:12,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37972 deadline: 1733584512313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:12,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:12,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38006 deadline: 1733584512313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:12,322 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:12,322 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:12,322 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:12,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:12,323 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:12,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38038 deadline: 1733584512313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:12,323 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:12,323 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:12,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:12,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:12,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:12,655 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=141 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/B/b030904bef4745bf93ddae147deb18da 2024-12-07T15:14:12,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/C/86b7d3d216f94081a38c6bf54d203b90 is 50, key is test_row_0/C:col10/1733584450448/Put/seqid=0 2024-12-07T15:14:12,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741979_1155 (size=12151) 2024-12-07T15:14:12,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-07T15:14:12,817 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:12,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38036 deadline: 1733584512813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:12,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:12,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:12,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:12,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:12,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38020 deadline: 1733584512813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:12,819 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:12,819 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:12,819 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:12,837 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:12,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37972 deadline: 1733584512833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:12,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:12,838 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:12,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38006 deadline: 1733584512833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:12,838 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:12,838 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:12,839 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:12,839 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:12,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:12,839 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:12,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38038 deadline: 1733584512833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:12,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:12,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:12,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:13,073 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=141 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/C/86b7d3d216f94081a38c6bf54d203b90 2024-12-07T15:14:13,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/A/a84ed360131f424999ab322f82d81d7e as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/a84ed360131f424999ab322f82d81d7e 2024-12-07T15:14:13,084 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/a84ed360131f424999ab322f82d81d7e, entries=150, sequenceid=141, filesize=30.4 K 2024-12-07T15:14:13,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/B/b030904bef4745bf93ddae147deb18da as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/b030904bef4745bf93ddae147deb18da 2024-12-07T15:14:13,089 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/b030904bef4745bf93ddae147deb18da, entries=150, sequenceid=141, filesize=11.9 K 2024-12-07T15:14:13,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/C/86b7d3d216f94081a38c6bf54d203b90 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/86b7d3d216f94081a38c6bf54d203b90 2024-12-07T15:14:13,095 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/86b7d3d216f94081a38c6bf54d203b90, entries=150, sequenceid=141, filesize=11.9 K 2024-12-07T15:14:13,096 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for b393b7989f53c0069cd9db33cd3a7e39 in 1696ms, sequenceid=141, compaction requested=true 2024-12-07T15:14:13,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for b393b7989f53c0069cd9db33cd3a7e39: 2024-12-07T15:14:13,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:13,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-12-07T15:14:13,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-12-07T15:14:13,100 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-12-07T15:14:13,100 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 4.4800 sec 2024-12-07T15:14:13,101 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 4.4860 sec 2024-12-07T15:14:13,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on b393b7989f53c0069cd9db33cd3a7e39 2024-12-07T15:14:13,854 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b393b7989f53c0069cd9db33cd3a7e39 3/3 column families, dataSize=140.89 KB heapSize=395.53 KB 2024-12-07T15:14:13,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b393b7989f53c0069cd9db33cd3a7e39, store=A 2024-12-07T15:14:13,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T15:14:13,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b393b7989f53c0069cd9db33cd3a7e39, store=B 2024-12-07T15:14:13,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T15:14:13,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b393b7989f53c0069cd9db33cd3a7e39, store=C 2024-12-07T15:14:13,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T15:14:13,863 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207657aec4b268f4761a72729173b828095_b393b7989f53c0069cd9db33cd3a7e39 is 50, key is test_row_0/A:col10/1733584451670/Put/seqid=0 2024-12-07T15:14:13,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741980_1156 (size=14794) 2024-12-07T15:14:13,869 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:13,869 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:13,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38038 deadline: 1733584513859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:13,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38006 deadline: 1733584513860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:13,870 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:13,870 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:13,870 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:13,870 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:13,870 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:13,870 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:13,872 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:13,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38036 deadline: 1733584513861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:13,873 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:13,873 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:13,873 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:13,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:13,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38020 deadline: 1733584513869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:13,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:13,874 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:13,874 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:13,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37972 deadline: 1733584513869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:13,874 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:13,878 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:13,878 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:13,878 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:13,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:13,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38038 deadline: 1733584513973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:13,976 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:13,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:13,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:13,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38006 deadline: 1733584513973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:13,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:13,976 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:13,976 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:13,976 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:13,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:13,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38036 deadline: 1733584513983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:13,984 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:13,984 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:13,984 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:13,984 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:13,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37972 deadline: 1733584513983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:13,984 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:13,984 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:13,984 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:13,984 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:13,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38020 deadline: 1733584513983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:13,985 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:13,985 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:13,985 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:14,186 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:14,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38038 deadline: 1733584514183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:14,187 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:14,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38006 deadline: 1733584514184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:14,187 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:14,187 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:14,187 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:14,187 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:14,187 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:14,187 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:14,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:14,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38036 deadline: 1733584514193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:14,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:14,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:14,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:14,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37972 deadline: 1733584514193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:14,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:14,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:14,195 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:14,195 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:14,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38020 deadline: 1733584514193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:14,195 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:14,195 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:14,196 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:14,196 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:14,268 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:14,273 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207657aec4b268f4761a72729173b828095_b393b7989f53c0069cd9db33cd3a7e39 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207657aec4b268f4761a72729173b828095_b393b7989f53c0069cd9db33cd3a7e39 2024-12-07T15:14:14,274 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/A/2d90f70a3fc04c049ca5385d8e4ced6c, store: [table=TestAcidGuarantees family=A region=b393b7989f53c0069cd9db33cd3a7e39] 2024-12-07T15:14:14,274 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/A/2d90f70a3fc04c049ca5385d8e4ced6c is 175, key is test_row_0/A:col10/1733584451670/Put/seqid=0 2024-12-07T15:14:14,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741981_1157 (size=39749) 2024-12-07T15:14:14,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:14,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38038 deadline: 1733584514493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:14,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:14,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:14,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38006 deadline: 1733584514493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:14,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:14,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:14,497 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:14,497 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:14,497 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:14,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:14,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38036 deadline: 1733584514503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:14,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:14,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:14,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:14,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:14,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37972 deadline: 1733584514503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:14,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:14,509 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:14,510 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:14,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38020 deadline: 1733584514503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:14,510 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:14,510 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:14,510 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:14,510 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:14,679 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=177, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/A/2d90f70a3fc04c049ca5385d8e4ced6c 2024-12-07T15:14:14,688 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/B/4ab5ecc506c24e14b8c05c9cfbd4cf7c is 50, key is test_row_0/B:col10/1733584451670/Put/seqid=0 2024-12-07T15:14:14,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741982_1158 (size=12151) 2024-12-07T15:14:15,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:15,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38038 deadline: 1733584515003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:15,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:15,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:15,005 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:15,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38006 deadline: 1733584515003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:15,005 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:15,005 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:15,005 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:15,005 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:15,014 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:15,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38036 deadline: 1733584515013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:15,014 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:15,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:15,015 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:15,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38020 deadline: 1733584515013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:15,015 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:15,015 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:15,015 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:15,015 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:15,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:15,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37972 deadline: 1733584515023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:15,030 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:15,030 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:15,030 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:15,093 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/B/4ab5ecc506c24e14b8c05c9cfbd4cf7c 2024-12-07T15:14:15,109 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/C/a1bada73e4894b859954823068ebd7f1 is 50, key is test_row_0/C:col10/1733584451670/Put/seqid=0 2024-12-07T15:14:15,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741983_1159 (size=12151) 2024-12-07T15:14:15,518 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/C/a1bada73e4894b859954823068ebd7f1 2024-12-07T15:14:15,524 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/A/2d90f70a3fc04c049ca5385d8e4ced6c as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/2d90f70a3fc04c049ca5385d8e4ced6c 2024-12-07T15:14:15,528 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/2d90f70a3fc04c049ca5385d8e4ced6c, entries=200, sequenceid=177, filesize=38.8 K 2024-12-07T15:14:15,529 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/B/4ab5ecc506c24e14b8c05c9cfbd4cf7c as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/4ab5ecc506c24e14b8c05c9cfbd4cf7c 2024-12-07T15:14:15,533 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/4ab5ecc506c24e14b8c05c9cfbd4cf7c, entries=150, sequenceid=177, filesize=11.9 K 2024-12-07T15:14:15,534 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/C/a1bada73e4894b859954823068ebd7f1 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/a1bada73e4894b859954823068ebd7f1 2024-12-07T15:14:15,540 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/a1bada73e4894b859954823068ebd7f1, entries=150, sequenceid=177, filesize=11.9 K 2024-12-07T15:14:15,541 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~395.48 KB/404976, currentSize=46.96 KB/48090 for b393b7989f53c0069cd9db33cd3a7e39 in 1686ms, sequenceid=177, compaction requested=true 2024-12-07T15:14:15,541 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b393b7989f53c0069cd9db33cd3a7e39: 2024-12-07T15:14:15,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b393b7989f53c0069cd9db33cd3a7e39:A, priority=-2147483648, current under compaction store size is 1 2024-12-07T15:14:15,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:14:15,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b393b7989f53c0069cd9db33cd3a7e39:B, priority=-2147483648, current under compaction store size is 2 2024-12-07T15:14:15,541 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-07T15:14:15,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:14:15,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b393b7989f53c0069cd9db33cd3a7e39:C, priority=-2147483648, current under compaction store size is 3 2024-12-07T15:14:15,541 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-07T15:14:15,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T15:14:15,542 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141564 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-07T15:14:15,542 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48510 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-07T15:14:15,542 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1541): b393b7989f53c0069cd9db33cd3a7e39/A is initiating minor compaction (all files) 2024-12-07T15:14:15,542 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1541): b393b7989f53c0069cd9db33cd3a7e39/B is initiating minor compaction (all files) 2024-12-07T15:14:15,542 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b393b7989f53c0069cd9db33cd3a7e39/A in TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:15,542 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b393b7989f53c0069cd9db33cd3a7e39/B in TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:15,543 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/c1a15f2e0802400c99c942950da40098, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/c5a156582a3446ad88d8c14897191d62, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/a84ed360131f424999ab322f82d81d7e, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/2d90f70a3fc04c049ca5385d8e4ced6c] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp, totalSize=138.2 K 2024-12-07T15:14:15,543 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:15,543 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/552958a789f94d69830a836c4e84f4bd, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/92eaac8ad8174f4a9fec575d3136cbc1, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/b030904bef4745bf93ddae147deb18da, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/4ab5ecc506c24e14b8c05c9cfbd4cf7c] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp, totalSize=47.4 K 2024-12-07T15:14:15,543 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. files: [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/c1a15f2e0802400c99c942950da40098, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/c5a156582a3446ad88d8c14897191d62, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/a84ed360131f424999ab322f82d81d7e, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/2d90f70a3fc04c049ca5385d8e4ced6c] 2024-12-07T15:14:15,543 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1a15f2e0802400c99c942950da40098, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733584449377 2024-12-07T15:14:15,543 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 552958a789f94d69830a836c4e84f4bd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733584449377 2024-12-07T15:14:15,543 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting c5a156582a3446ad88d8c14897191d62, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1733584449779 2024-12-07T15:14:15,543 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 92eaac8ad8174f4a9fec575d3136cbc1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1733584449789 2024-12-07T15:14:15,544 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting b030904bef4745bf93ddae147deb18da, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1733584450448 2024-12-07T15:14:15,544 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting a84ed360131f424999ab322f82d81d7e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1733584450448 2024-12-07T15:14:15,544 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ab5ecc506c24e14b8c05c9cfbd4cf7c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1733584451655 2024-12-07T15:14:15,544 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d90f70a3fc04c049ca5385d8e4ced6c, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1733584451655 2024-12-07T15:14:15,557 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b393b7989f53c0069cd9db33cd3a7e39#B#compaction#134 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:14:15,558 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/B/1f26df873f9e4232880a41c38086017c is 50, key is test_row_0/B:col10/1733584451670/Put/seqid=0 2024-12-07T15:14:15,561 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b393b7989f53c0069cd9db33cd3a7e39] 2024-12-07T15:14:15,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741984_1160 (size=12493) 2024-12-07T15:14:15,569 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412076750f423c2974b16a9d8d84717260f98_b393b7989f53c0069cd9db33cd3a7e39 store=[table=TestAcidGuarantees family=A region=b393b7989f53c0069cd9db33cd3a7e39] 2024-12-07T15:14:15,571 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/B/1f26df873f9e4232880a41c38086017c as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/1f26df873f9e4232880a41c38086017c 2024-12-07T15:14:15,572 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412076750f423c2974b16a9d8d84717260f98_b393b7989f53c0069cd9db33cd3a7e39, store=[table=TestAcidGuarantees family=A region=b393b7989f53c0069cd9db33cd3a7e39] 2024-12-07T15:14:15,573 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412076750f423c2974b16a9d8d84717260f98_b393b7989f53c0069cd9db33cd3a7e39 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b393b7989f53c0069cd9db33cd3a7e39] 2024-12-07T15:14:15,579 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b393b7989f53c0069cd9db33cd3a7e39/B of b393b7989f53c0069cd9db33cd3a7e39 into 1f26df873f9e4232880a41c38086017c(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:14:15,579 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b393b7989f53c0069cd9db33cd3a7e39: 2024-12-07T15:14:15,580 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., storeName=b393b7989f53c0069cd9db33cd3a7e39/B, priority=12, startTime=1733584455541; duration=0sec 2024-12-07T15:14:15,580 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T15:14:15,580 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b393b7989f53c0069cd9db33cd3a7e39:B 2024-12-07T15:14:15,580 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-07T15:14:15,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741985_1161 (size=4469) 2024-12-07T15:14:15,581 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48510 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-07T15:14:15,582 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1541): b393b7989f53c0069cd9db33cd3a7e39/C is initiating minor compaction (all files) 2024-12-07T15:14:15,582 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b393b7989f53c0069cd9db33cd3a7e39/C in TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:15,582 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/a247c9568ce542f2a817d6e46be55dc1, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/9141de3767344ab09d9b1c671525f76a, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/86b7d3d216f94081a38c6bf54d203b90, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/a1bada73e4894b859954823068ebd7f1] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp, totalSize=47.4 K 2024-12-07T15:14:15,582 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b393b7989f53c0069cd9db33cd3a7e39#A#compaction#135 average throughput is 1.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:14:15,583 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/A/ef20962a0815447a937e2beaec252bb7 is 175, key is test_row_0/A:col10/1733584451670/Put/seqid=0 2024-12-07T15:14:15,584 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting a247c9568ce542f2a817d6e46be55dc1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733584449377 2024-12-07T15:14:15,584 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 9141de3767344ab09d9b1c671525f76a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1733584449789 2024-12-07T15:14:15,585 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 86b7d3d216f94081a38c6bf54d203b90, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1733584450448 2024-12-07T15:14:15,585 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting a1bada73e4894b859954823068ebd7f1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1733584451655 2024-12-07T15:14:15,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741986_1162 (size=31447) 2024-12-07T15:14:15,599 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b393b7989f53c0069cd9db33cd3a7e39#C#compaction#136 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:14:15,600 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/C/6868d4bd16bd4a6ba9f4b0314fc71a97 is 50, key is test_row_0/C:col10/1733584451670/Put/seqid=0 2024-12-07T15:14:15,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741987_1163 (size=12493) 2024-12-07T15:14:15,995 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/A/ef20962a0815447a937e2beaec252bb7 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/ef20962a0815447a937e2beaec252bb7 2024-12-07T15:14:16,001 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b393b7989f53c0069cd9db33cd3a7e39/A of b393b7989f53c0069cd9db33cd3a7e39 into ef20962a0815447a937e2beaec252bb7(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:14:16,001 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b393b7989f53c0069cd9db33cd3a7e39: 2024-12-07T15:14:16,001 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., storeName=b393b7989f53c0069cd9db33cd3a7e39/A, priority=12, startTime=1733584455541; duration=0sec 2024-12-07T15:14:16,001 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:14:16,001 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b393b7989f53c0069cd9db33cd3a7e39:A 2024-12-07T15:14:16,010 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/C/6868d4bd16bd4a6ba9f4b0314fc71a97 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/6868d4bd16bd4a6ba9f4b0314fc71a97 2024-12-07T15:14:16,021 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b393b7989f53c0069cd9db33cd3a7e39/C of b393b7989f53c0069cd9db33cd3a7e39 into 6868d4bd16bd4a6ba9f4b0314fc71a97(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:14:16,021 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b393b7989f53c0069cd9db33cd3a7e39: 2024-12-07T15:14:16,021 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., storeName=b393b7989f53c0069cd9db33cd3a7e39/C, priority=12, startTime=1733584455541; duration=0sec 2024-12-07T15:14:16,022 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:14:16,022 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b393b7989f53c0069cd9db33cd3a7e39:C 2024-12-07T15:14:16,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on b393b7989f53c0069cd9db33cd3a7e39 2024-12-07T15:14:16,022 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b393b7989f53c0069cd9db33cd3a7e39 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-07T15:14:16,022 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b393b7989f53c0069cd9db33cd3a7e39, store=A 2024-12-07T15:14:16,022 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:14:16,022 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b393b7989f53c0069cd9db33cd3a7e39, store=B 2024-12-07T15:14:16,022 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:14:16,022 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b393b7989f53c0069cd9db33cd3a7e39, store=C 2024-12-07T15:14:16,022 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:14:16,032 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412072d7ab2765ff546048b30348287f609fb_b393b7989f53c0069cd9db33cd3a7e39 is 50, key is test_row_0/A:col10/1733584453856/Put/seqid=0 2024-12-07T15:14:16,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741988_1164 (size=14794) 2024-12-07T15:14:16,042 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:16,046 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412072d7ab2765ff546048b30348287f609fb_b393b7989f53c0069cd9db33cd3a7e39 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412072d7ab2765ff546048b30348287f609fb_b393b7989f53c0069cd9db33cd3a7e39 2024-12-07T15:14:16,047 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/A/cfff55a4dae24ae698d87826c790b908, store: [table=TestAcidGuarantees family=A region=b393b7989f53c0069cd9db33cd3a7e39] 2024-12-07T15:14:16,048 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/A/cfff55a4dae24ae698d87826c790b908 is 175, key is test_row_0/A:col10/1733584453856/Put/seqid=0 2024-12-07T15:14:16,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741989_1165 (size=39749) 2024-12-07T15:14:16,061 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733584456046, max timestamp=1733584456057Num uniques 859; 2024-12-07T15:14:16,062 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-07T15:14:16,062 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733584456046, max timestamp=1733584456058Num uniques 859; 2024-12-07T15:14:16,062 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-07T15:14:16,062 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733584456046, max timestamp=1733584456057Num uniques 859; 2024-12-07T15:14:16,063 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-07T15:14:16,081 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:16,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38038 deadline: 1733584516077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:16,082 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:16,082 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:16,082 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:16,086 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:16,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38036 deadline: 1733584516077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:16,087 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:16,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:16,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38006 deadline: 1733584516079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:16,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:16,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:16,087 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:16,087 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:16,087 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:16,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37972 deadline: 1733584516080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:16,087 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:16,088 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:16,088 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:16,088 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:16,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38020 deadline: 1733584516081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:16,088 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:16,088 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:16,088 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:16,088 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:16,198 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:16,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38038 deadline: 1733584516193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:16,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:16,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:16,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:16,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38036 deadline: 1733584516193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:16,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:16,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:16,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:16,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:16,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38006 deadline: 1733584516193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:16,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:16,199 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:16,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:16,200 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:16,200 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:16,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37972 deadline: 1733584516193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:16,200 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:16,200 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:16,200 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:16,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38020 deadline: 1733584516193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:16,200 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:16,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:16,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:16,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:16,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:16,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38038 deadline: 1733584516403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:16,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:16,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38036 deadline: 1733584516403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:16,410 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:16,410 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:16,410 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:16,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:16,410 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:16,410 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:16,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38006 deadline: 1733584516403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:16,410 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:16,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:16,410 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:16,410 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:16,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37972 deadline: 1733584516403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:16,410 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:16,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:16,411 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:16,411 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:16,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38020 deadline: 1733584516403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:16,411 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:16,411 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:16,411 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:16,411 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:16,459 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=191, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/A/cfff55a4dae24ae698d87826c790b908 2024-12-07T15:14:16,467 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/B/1f2306b2566d486aa7b3171708f6f098 is 50, key is test_row_0/B:col10/1733584453856/Put/seqid=0 2024-12-07T15:14:16,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741990_1166 (size=12151) 2024-12-07T15:14:16,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:16,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38038 deadline: 1733584516713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:16,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:16,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:16,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:16,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38036 deadline: 1733584516713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:16,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:16,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:16,718 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:16,718 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:16,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38006 deadline: 1733584516713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:16,718 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:16,718 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:16,718 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:16,718 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:16,724 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:16,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37972 deadline: 1733584516723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:16,724 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:16,724 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:16,724 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:16,724 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:16,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38020 deadline: 1733584516723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:16,725 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:16,725 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:16,725 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:16,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-07T15:14:16,813 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T15:14:16,814 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T15:14:16,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-12-07T15:14:16,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-07T15:14:16,816 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T15:14:16,817 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T15:14:16,817 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T15:14:16,878 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/B/1f2306b2566d486aa7b3171708f6f098 2024-12-07T15:14:16,887 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/C/920149c45f5d442b900068d7311c8a22 is 50, key is test_row_0/C:col10/1733584453856/Put/seqid=0 2024-12-07T15:14:16,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741991_1167 (size=12151) 2024-12-07T15:14:16,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-07T15:14:16,969 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-07T15:14:16,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:16,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. as already flushing 2024-12-07T15:14:16,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:16,970 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:16,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:16,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:17,123 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-07T15:14:17,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:17,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. as already flushing 2024-12-07T15:14:17,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:17,123 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:17,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:17,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:17,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-07T15:14:17,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:17,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38038 deadline: 1733584517223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:17,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:17,226 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:17,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38036 deadline: 1733584517223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:17,226 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:17,226 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:17,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:17,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:17,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:17,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:17,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38006 deadline: 1733584517223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:17,228 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:17,228 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:17,228 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:17,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:17,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37972 deadline: 1733584517234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:17,237 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:17,237 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:17,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38020 deadline: 1733584517234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:17,237 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:17,237 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:17,237 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:17,237 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:17,237 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:17,276 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-07T15:14:17,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:17,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. as already flushing 2024-12-07T15:14:17,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:17,277 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:17,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:17,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:17,291 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/C/920149c45f5d442b900068d7311c8a22 2024-12-07T15:14:17,297 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/A/cfff55a4dae24ae698d87826c790b908 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/cfff55a4dae24ae698d87826c790b908 2024-12-07T15:14:17,301 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/cfff55a4dae24ae698d87826c790b908, entries=200, sequenceid=191, filesize=38.8 K 2024-12-07T15:14:17,302 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/B/1f2306b2566d486aa7b3171708f6f098 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/1f2306b2566d486aa7b3171708f6f098 2024-12-07T15:14:17,306 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/1f2306b2566d486aa7b3171708f6f098, entries=150, sequenceid=191, filesize=11.9 K 2024-12-07T15:14:17,307 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/C/920149c45f5d442b900068d7311c8a22 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/920149c45f5d442b900068d7311c8a22 2024-12-07T15:14:17,314 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/920149c45f5d442b900068d7311c8a22, entries=150, sequenceid=191, filesize=11.9 K 2024-12-07T15:14:17,316 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for b393b7989f53c0069cd9db33cd3a7e39 in 1294ms, sequenceid=191, compaction requested=false 2024-12-07T15:14:17,316 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b393b7989f53c0069cd9db33cd3a7e39: 2024-12-07T15:14:17,429 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-07T15:14:17,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:17,430 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing b393b7989f53c0069cd9db33cd3a7e39 3/3 column families, dataSize=134.18 KB heapSize=370.92 KB 2024-12-07T15:14:17,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b393b7989f53c0069cd9db33cd3a7e39, store=A 2024-12-07T15:14:17,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T15:14:17,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b393b7989f53c0069cd9db33cd3a7e39, store=B 2024-12-07T15:14:17,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T15:14:17,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b393b7989f53c0069cd9db33cd3a7e39, store=C 2024-12-07T15:14:17,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T15:14:17,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207eda85a80ba934844a3cacc61d63c1bed_b393b7989f53c0069cd9db33cd3a7e39 is 50, key is test_row_0/A:col10/1733584456076/Put/seqid=0 2024-12-07T15:14:17,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-07T15:14:17,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741992_1168 (size=12304) 2024-12-07T15:14:17,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:17,855 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207eda85a80ba934844a3cacc61d63c1bed_b393b7989f53c0069cd9db33cd3a7e39 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207eda85a80ba934844a3cacc61d63c1bed_b393b7989f53c0069cd9db33cd3a7e39 2024-12-07T15:14:17,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/A/247419885c4646a5b651a79b07005e99, store: [table=TestAcidGuarantees family=A region=b393b7989f53c0069cd9db33cd3a7e39] 2024-12-07T15:14:17,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/A/247419885c4646a5b651a79b07005e99 is 175, key is test_row_0/A:col10/1733584456076/Put/seqid=0 2024-12-07T15:14:17,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741993_1169 (size=31105) 2024-12-07T15:14:17,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-07T15:14:18,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on b393b7989f53c0069cd9db33cd3a7e39 2024-12-07T15:14:18,238 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. as already flushing 2024-12-07T15:14:18,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:18,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38038 deadline: 1733584518252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:18,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:18,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:18,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37972 deadline: 1733584518253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:18,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:18,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:18,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:18,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38020 deadline: 1733584518253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:18,258 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to d6797ebe9c52,46009,1733584418401 for 'test_row_2' in b393b7989f53c0069cd9db33cd3a7e39 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-07T15:14:18,258 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to d6797ebe9c52,46009,1733584418401 for 'test_row_1' in b393b7989f53c0069cd9db33cd3a7e39 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-07T15:14:18,259 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:18,259 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:18,259 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:18,259 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:18,259 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:18,259 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:18,262 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=225, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/A/247419885c4646a5b651a79b07005e99 2024-12-07T15:14:18,265 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:18,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:18,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38036 deadline: 1733584518257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:18,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38006 deadline: 1733584518257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:18,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:18,265 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:18,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:18,265 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:18,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:18,265 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:18,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/B/88594334250a43ec8fca86bb43720f2a is 50, key is test_row_0/B:col10/1733584456076/Put/seqid=0 2024-12-07T15:14:18,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741994_1170 (size=12151) 2024-12-07T15:14:18,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:18,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38038 deadline: 1733584518363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:18,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:18,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:18,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:18,373 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:18,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38036 deadline: 1733584518373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:18,374 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:18,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:18,374 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:18,374 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:18,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38006 deadline: 1733584518373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:18,375 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:18,375 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:18,375 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:18,580 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:18,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38038 deadline: 1733584518573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:18,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:18,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:18,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:18,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:18,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38036 deadline: 1733584518583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:18,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:18,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:18,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38006 deadline: 1733584518583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:18,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:18,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:18,587 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:18,587 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:18,587 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:18,676 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/B/88594334250a43ec8fca86bb43720f2a 2024-12-07T15:14:18,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/C/fb76d6ae7a1e4924b2ee5721322c60a9 is 50, key is test_row_0/C:col10/1733584456076/Put/seqid=0 2024-12-07T15:14:18,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741995_1171 (size=12151) 2024-12-07T15:14:18,893 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:18,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38038 deadline: 1733584518893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:18,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:18,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38036 deadline: 1733584518893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:18,894 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:18,894 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:18,894 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:18,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:18,894 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:18,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38006 deadline: 1733584518893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:18,894 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:18,894 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:18,894 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:18,894 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:18,894 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:18,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-07T15:14:19,091 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/C/fb76d6ae7a1e4924b2ee5721322c60a9 2024-12-07T15:14:19,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/A/247419885c4646a5b651a79b07005e99 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/247419885c4646a5b651a79b07005e99 2024-12-07T15:14:19,101 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/247419885c4646a5b651a79b07005e99, entries=150, sequenceid=225, filesize=30.4 K 2024-12-07T15:14:19,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/B/88594334250a43ec8fca86bb43720f2a as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/88594334250a43ec8fca86bb43720f2a 2024-12-07T15:14:19,105 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/88594334250a43ec8fca86bb43720f2a, entries=150, sequenceid=225, filesize=11.9 K 2024-12-07T15:14:19,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/C/fb76d6ae7a1e4924b2ee5721322c60a9 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/fb76d6ae7a1e4924b2ee5721322c60a9 2024-12-07T15:14:19,111 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/fb76d6ae7a1e4924b2ee5721322c60a9, entries=150, sequenceid=225, filesize=11.9 K 2024-12-07T15:14:19,112 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~370.88 KB/379776, currentSize=53.67 KB/54960 for b393b7989f53c0069cd9db33cd3a7e39 in 1681ms, sequenceid=225, compaction requested=true 2024-12-07T15:14:19,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for b393b7989f53c0069cd9db33cd3a7e39: 2024-12-07T15:14:19,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:19,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-12-07T15:14:19,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-12-07T15:14:19,116 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-12-07T15:14:19,116 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2960 sec 2024-12-07T15:14:19,118 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 2.3030 sec 2024-12-07T15:14:19,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on b393b7989f53c0069cd9db33cd3a7e39 2024-12-07T15:14:19,405 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b393b7989f53c0069cd9db33cd3a7e39 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-07T15:14:19,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b393b7989f53c0069cd9db33cd3a7e39, store=A 2024-12-07T15:14:19,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:14:19,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b393b7989f53c0069cd9db33cd3a7e39, store=B 2024-12-07T15:14:19,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:14:19,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b393b7989f53c0069cd9db33cd3a7e39, store=C 2024-12-07T15:14:19,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:14:19,417 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207835dc7cd66644d81aec4f28f08117fa4_b393b7989f53c0069cd9db33cd3a7e39 is 50, key is test_row_0/A:col10/1733584459405/Put/seqid=0 2024-12-07T15:14:19,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741996_1172 (size=14794) 2024-12-07T15:14:19,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:19,438 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:19,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38038 deadline: 1733584519432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:19,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38036 deadline: 1733584519434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:19,438 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:19,438 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:19,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:19,438 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:19,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:19,439 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:19,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:19,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38006 deadline: 1733584519438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:19,441 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:19,441 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:19,441 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:19,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:19,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38038 deadline: 1733584519543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:19,550 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:19,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:19,550 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:19,550 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:19,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38036 deadline: 1733584519543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:19,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:19,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:19,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:19,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:19,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38006 deadline: 1733584519543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:19,551 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:19,551 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:19,551 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:19,756 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:19,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38038 deadline: 1733584519753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:19,756 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:19,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38006 deadline: 1733584519753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:19,757 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:19,757 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:19,757 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:19,757 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:19,757 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:19,757 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:19,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:19,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38036 deadline: 1733584519763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:19,770 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:19,770 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:19,770 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:19,825 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:19,830 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207835dc7cd66644d81aec4f28f08117fa4_b393b7989f53c0069cd9db33cd3a7e39 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207835dc7cd66644d81aec4f28f08117fa4_b393b7989f53c0069cd9db33cd3a7e39 2024-12-07T15:14:19,831 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/A/b76cfbd63a1c4c3cac9af7473155a27d, store: [table=TestAcidGuarantees family=A region=b393b7989f53c0069cd9db33cd3a7e39] 2024-12-07T15:14:19,831 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/A/b76cfbd63a1c4c3cac9af7473155a27d is 175, key is test_row_0/A:col10/1733584459405/Put/seqid=0 2024-12-07T15:14:19,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741997_1173 (size=39749) 2024-12-07T15:14:19,837 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=238, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/A/b76cfbd63a1c4c3cac9af7473155a27d 2024-12-07T15:14:19,845 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/B/ec627eac599e4eaaab83d12575fc00ce is 50, key is test_row_0/B:col10/1733584459405/Put/seqid=0 2024-12-07T15:14:19,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741998_1174 (size=12151) 2024-12-07T15:14:19,852 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/B/ec627eac599e4eaaab83d12575fc00ce 2024-12-07T15:14:19,863 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/C/dacfa487b0ca4b08a1c5a53762bc4581 is 50, key is test_row_0/C:col10/1733584459405/Put/seqid=0 2024-12-07T15:14:19,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741999_1175 (size=12151) 2024-12-07T15:14:20,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:20,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38006 deadline: 1733584520064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:20,067 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:20,067 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:20,067 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:20,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38038 deadline: 1733584520064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:20,067 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:20,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:20,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:20,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:20,086 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:20,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38036 deadline: 1733584520083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:20,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:20,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:20,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:20,268 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/C/dacfa487b0ca4b08a1c5a53762bc4581 2024-12-07T15:14:20,274 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/A/b76cfbd63a1c4c3cac9af7473155a27d as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/b76cfbd63a1c4c3cac9af7473155a27d 2024-12-07T15:14:20,274 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:20,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37972 deadline: 1733584520273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:20,275 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to d6797ebe9c52,46009,1733584418401 for 'test_row_1' in b393b7989f53c0069cd9db33cd3a7e39 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-07T15:14:20,276 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:20,276 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:20,276 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:20,278 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/b76cfbd63a1c4c3cac9af7473155a27d, entries=200, sequenceid=238, filesize=38.8 K 2024-12-07T15:14:20,279 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/B/ec627eac599e4eaaab83d12575fc00ce as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/ec627eac599e4eaaab83d12575fc00ce 2024-12-07T15:14:20,284 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/ec627eac599e4eaaab83d12575fc00ce, entries=150, sequenceid=238, filesize=11.9 K 2024-12-07T15:14:20,284 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:20,285 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/C/dacfa487b0ca4b08a1c5a53762bc4581 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/dacfa487b0ca4b08a1c5a53762bc4581 2024-12-07T15:14:20,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38020 deadline: 1733584520283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:20,285 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to d6797ebe9c52,46009,1733584418401 for 'test_row_2' in b393b7989f53c0069cd9db33cd3a7e39 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4203 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-07T15:14:20,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:20,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:20,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:20,289 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/dacfa487b0ca4b08a1c5a53762bc4581, entries=150, sequenceid=238, filesize=11.9 K 2024-12-07T15:14:20,289 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for b393b7989f53c0069cd9db33cd3a7e39 in 884ms, sequenceid=238, compaction requested=true 2024-12-07T15:14:20,289 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b393b7989f53c0069cd9db33cd3a7e39: 2024-12-07T15:14:20,290 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b393b7989f53c0069cd9db33cd3a7e39:A, priority=-2147483648, current under compaction store size is 1 2024-12-07T15:14:20,290 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:14:20,290 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-07T15:14:20,290 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b393b7989f53c0069cd9db33cd3a7e39:B, priority=-2147483648, current under compaction store size is 2 2024-12-07T15:14:20,290 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:14:20,290 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b393b7989f53c0069cd9db33cd3a7e39:C, priority=-2147483648, current under compaction store size is 3 2024-12-07T15:14:20,290 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-07T15:14:20,290 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T15:14:20,291 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142050 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-07T15:14:20,291 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1541): b393b7989f53c0069cd9db33cd3a7e39/A is initiating minor compaction (all files) 2024-12-07T15:14:20,291 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-07T15:14:20,291 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b393b7989f53c0069cd9db33cd3a7e39/A in TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:20,291 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1541): b393b7989f53c0069cd9db33cd3a7e39/B is initiating minor compaction (all files) 2024-12-07T15:14:20,292 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b393b7989f53c0069cd9db33cd3a7e39/B in TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:20,292 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/ef20962a0815447a937e2beaec252bb7, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/cfff55a4dae24ae698d87826c790b908, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/247419885c4646a5b651a79b07005e99, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/b76cfbd63a1c4c3cac9af7473155a27d] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp, totalSize=138.7 K 2024-12-07T15:14:20,292 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:20,292 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/1f26df873f9e4232880a41c38086017c, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/1f2306b2566d486aa7b3171708f6f098, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/88594334250a43ec8fca86bb43720f2a, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/ec627eac599e4eaaab83d12575fc00ce] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp, totalSize=47.8 K 2024-12-07T15:14:20,292 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. files: [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/ef20962a0815447a937e2beaec252bb7, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/cfff55a4dae24ae698d87826c790b908, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/247419885c4646a5b651a79b07005e99, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/b76cfbd63a1c4c3cac9af7473155a27d] 2024-12-07T15:14:20,293 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f26df873f9e4232880a41c38086017c, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1733584451655 2024-12-07T15:14:20,293 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef20962a0815447a937e2beaec252bb7, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1733584451655 2024-12-07T15:14:20,293 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f2306b2566d486aa7b3171708f6f098, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1733584453856 2024-12-07T15:14:20,293 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting cfff55a4dae24ae698d87826c790b908, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1733584453856 2024-12-07T15:14:20,293 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 88594334250a43ec8fca86bb43720f2a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1733584456076 2024-12-07T15:14:20,293 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 247419885c4646a5b651a79b07005e99, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1733584456076 2024-12-07T15:14:20,294 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting ec627eac599e4eaaab83d12575fc00ce, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1733584458248 2024-12-07T15:14:20,294 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting b76cfbd63a1c4c3cac9af7473155a27d, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1733584458248 2024-12-07T15:14:20,303 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b393b7989f53c0069cd9db33cd3a7e39#B#compaction#146 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:14:20,304 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/B/e000b6d727814135aa3c6b38b2758961 is 50, key is test_row_0/B:col10/1733584459405/Put/seqid=0 2024-12-07T15:14:20,307 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b393b7989f53c0069cd9db33cd3a7e39] 2024-12-07T15:14:20,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742000_1176 (size=12629) 2024-12-07T15:14:20,313 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241207bcad39280b674dc1bf47dd210e0f88a7_b393b7989f53c0069cd9db33cd3a7e39 store=[table=TestAcidGuarantees family=A region=b393b7989f53c0069cd9db33cd3a7e39] 2024-12-07T15:14:20,316 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241207bcad39280b674dc1bf47dd210e0f88a7_b393b7989f53c0069cd9db33cd3a7e39, store=[table=TestAcidGuarantees family=A region=b393b7989f53c0069cd9db33cd3a7e39] 2024-12-07T15:14:20,316 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207bcad39280b674dc1bf47dd210e0f88a7_b393b7989f53c0069cd9db33cd3a7e39 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b393b7989f53c0069cd9db33cd3a7e39] 2024-12-07T15:14:20,321 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/B/e000b6d727814135aa3c6b38b2758961 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/e000b6d727814135aa3c6b38b2758961 2024-12-07T15:14:20,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742001_1177 (size=4469) 2024-12-07T15:14:20,327 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b393b7989f53c0069cd9db33cd3a7e39/B of b393b7989f53c0069cd9db33cd3a7e39 into e000b6d727814135aa3c6b38b2758961(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:14:20,327 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b393b7989f53c0069cd9db33cd3a7e39: 2024-12-07T15:14:20,327 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., storeName=b393b7989f53c0069cd9db33cd3a7e39/B, priority=12, startTime=1733584460290; duration=0sec 2024-12-07T15:14:20,327 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T15:14:20,327 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b393b7989f53c0069cd9db33cd3a7e39:B 2024-12-07T15:14:20,327 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-07T15:14:20,329 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-07T15:14:20,329 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1541): b393b7989f53c0069cd9db33cd3a7e39/C is initiating minor compaction (all files) 2024-12-07T15:14:20,329 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b393b7989f53c0069cd9db33cd3a7e39/C in TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:20,329 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/6868d4bd16bd4a6ba9f4b0314fc71a97, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/920149c45f5d442b900068d7311c8a22, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/fb76d6ae7a1e4924b2ee5721322c60a9, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/dacfa487b0ca4b08a1c5a53762bc4581] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp, totalSize=47.8 K 2024-12-07T15:14:20,329 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 6868d4bd16bd4a6ba9f4b0314fc71a97, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1733584451655 2024-12-07T15:14:20,330 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 920149c45f5d442b900068d7311c8a22, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1733584453856 2024-12-07T15:14:20,330 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting fb76d6ae7a1e4924b2ee5721322c60a9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1733584456076 2024-12-07T15:14:20,330 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting dacfa487b0ca4b08a1c5a53762bc4581, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1733584458248 2024-12-07T15:14:20,339 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b393b7989f53c0069cd9db33cd3a7e39#C#compaction#148 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:14:20,339 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/C/96f8121fa74b460eb6249314f02d8c6b is 50, key is test_row_0/C:col10/1733584459405/Put/seqid=0 2024-12-07T15:14:20,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742002_1178 (size=12629) 2024-12-07T15:14:20,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on b393b7989f53c0069cd9db33cd3a7e39 2024-12-07T15:14:20,577 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b393b7989f53c0069cd9db33cd3a7e39 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-07T15:14:20,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b393b7989f53c0069cd9db33cd3a7e39, store=A 2024-12-07T15:14:20,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T15:14:20,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b393b7989f53c0069cd9db33cd3a7e39, store=B 2024-12-07T15:14:20,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T15:14:20,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b393b7989f53c0069cd9db33cd3a7e39, store=C 2024-12-07T15:14:20,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T15:14:20,589 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412070134dacf0f68428685c7b670ff910c3f_b393b7989f53c0069cd9db33cd3a7e39 is 50, key is test_row_0/A:col10/1733584460577/Put/seqid=0 2024-12-07T15:14:20,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742003_1179 (size=14894) 2024-12-07T15:14:20,603 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:20,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:20,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38038 deadline: 1733584520596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:20,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38006 deadline: 1733584520597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:20,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:20,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38036 deadline: 1733584520598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:20,603 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:20,603 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:20,603 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:20,604 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:20,604 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:20,604 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:20,604 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:20,604 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:20,604 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:20,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:20,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38038 deadline: 1733584520713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:20,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:20,716 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:20,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38006 deadline: 1733584520713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:20,716 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:20,716 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:20,716 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:20,716 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:20,716 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:20,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38036 deadline: 1733584520713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:20,716 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:20,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:20,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:20,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:20,724 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b393b7989f53c0069cd9db33cd3a7e39#A#compaction#147 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:14:20,725 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/A/1c61771a38004224bd44cf2faa552483 is 175, key is test_row_0/A:col10/1733584459405/Put/seqid=0 2024-12-07T15:14:20,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742004_1180 (size=31583) 2024-12-07T15:14:20,752 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/C/96f8121fa74b460eb6249314f02d8c6b as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/96f8121fa74b460eb6249314f02d8c6b 2024-12-07T15:14:20,757 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b393b7989f53c0069cd9db33cd3a7e39/C of b393b7989f53c0069cd9db33cd3a7e39 into 96f8121fa74b460eb6249314f02d8c6b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:14:20,757 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b393b7989f53c0069cd9db33cd3a7e39: 2024-12-07T15:14:20,757 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., storeName=b393b7989f53c0069cd9db33cd3a7e39/C, priority=12, startTime=1733584460290; duration=0sec 2024-12-07T15:14:20,757 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:14:20,757 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b393b7989f53c0069cd9db33cd3a7e39:C 2024-12-07T15:14:20,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:20,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38038 deadline: 1733584520923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:20,924 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:20,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:20,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38006 deadline: 1733584520923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:20,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:20,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:20,924 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:20,925 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:20,925 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:20,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38036 deadline: 1733584520923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:20,925 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:20,925 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:20,925 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:20,925 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:20,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-07T15:14:20,973 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T15:14:20,974 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T15:14:20,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-12-07T15:14:20,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-07T15:14:20,976 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T15:14:20,977 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T15:14:20,977 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T15:14:20,994 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:20,998 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412070134dacf0f68428685c7b670ff910c3f_b393b7989f53c0069cd9db33cd3a7e39 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412070134dacf0f68428685c7b670ff910c3f_b393b7989f53c0069cd9db33cd3a7e39 2024-12-07T15:14:21,000 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/A/73d8a1eff7a34169ad538f47feb9405e, store: [table=TestAcidGuarantees family=A region=b393b7989f53c0069cd9db33cd3a7e39] 2024-12-07T15:14:21,000 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/A/73d8a1eff7a34169ad538f47feb9405e is 175, key is test_row_0/A:col10/1733584460577/Put/seqid=0 2024-12-07T15:14:21,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742005_1181 (size=39849) 2024-12-07T15:14:21,005 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=260, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/A/73d8a1eff7a34169ad538f47feb9405e 2024-12-07T15:14:21,013 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/B/6e18442b5c234b2397e183e20d5aff63 is 50, key is test_row_0/B:col10/1733584460577/Put/seqid=0 2024-12-07T15:14:21,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742006_1182 (size=12201) 2024-12-07T15:14:21,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-07T15:14:21,109 INFO [master/d6797ebe9c52:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-07T15:14:21,109 INFO [master/d6797ebe9c52:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-07T15:14:21,143 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-07T15:14:21,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:21,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. as already flushing 2024-12-07T15:14:21,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:21,143 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:21,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:21,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:21,148 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/A/1c61771a38004224bd44cf2faa552483 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/1c61771a38004224bd44cf2faa552483 2024-12-07T15:14:21,153 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b393b7989f53c0069cd9db33cd3a7e39/A of b393b7989f53c0069cd9db33cd3a7e39 into 1c61771a38004224bd44cf2faa552483(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:14:21,153 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b393b7989f53c0069cd9db33cd3a7e39: 2024-12-07T15:14:21,153 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., storeName=b393b7989f53c0069cd9db33cd3a7e39/A, priority=12, startTime=1733584460290; duration=0sec 2024-12-07T15:14:21,154 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:14:21,154 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b393b7989f53c0069cd9db33cd3a7e39:A 2024-12-07T15:14:21,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:21,235 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:21,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38006 deadline: 1733584521233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:21,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38038 deadline: 1733584521233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:21,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:21,236 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:21,236 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:21,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38036 deadline: 1733584521234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:21,236 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:21,236 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:21,236 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:21,236 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:21,236 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:21,236 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:21,236 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:21,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-07T15:14:21,297 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-07T15:14:21,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:21,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. as already flushing 2024-12-07T15:14:21,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:21,297 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:21,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:21,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:21,418 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/B/6e18442b5c234b2397e183e20d5aff63 2024-12-07T15:14:21,431 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/C/5650c4e397e245c0aff67bca28e594e9 is 50, key is test_row_0/C:col10/1733584460577/Put/seqid=0 2024-12-07T15:14:21,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742007_1183 (size=12201) 2024-12-07T15:14:21,438 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/C/5650c4e397e245c0aff67bca28e594e9 2024-12-07T15:14:21,445 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/A/73d8a1eff7a34169ad538f47feb9405e as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/73d8a1eff7a34169ad538f47feb9405e 2024-12-07T15:14:21,450 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-07T15:14:21,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:21,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. as already flushing 2024-12-07T15:14:21,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:21,451 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:21,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:21,451 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/73d8a1eff7a34169ad538f47feb9405e, entries=200, sequenceid=260, filesize=38.9 K 2024-12-07T15:14:21,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:21,453 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/B/6e18442b5c234b2397e183e20d5aff63 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/6e18442b5c234b2397e183e20d5aff63 2024-12-07T15:14:21,458 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/6e18442b5c234b2397e183e20d5aff63, entries=150, sequenceid=260, filesize=11.9 K 2024-12-07T15:14:21,459 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/C/5650c4e397e245c0aff67bca28e594e9 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/5650c4e397e245c0aff67bca28e594e9 2024-12-07T15:14:21,465 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/5650c4e397e245c0aff67bca28e594e9, entries=150, sequenceid=260, filesize=11.9 K 2024-12-07T15:14:21,468 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for b393b7989f53c0069cd9db33cd3a7e39 in 891ms, sequenceid=260, compaction requested=false 2024-12-07T15:14:21,468 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b393b7989f53c0069cd9db33cd3a7e39: 2024-12-07T15:14:21,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-07T15:14:21,604 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-07T15:14:21,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:21,605 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing b393b7989f53c0069cd9db33cd3a7e39 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-07T15:14:21,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b393b7989f53c0069cd9db33cd3a7e39, store=A 2024-12-07T15:14:21,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:14:21,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b393b7989f53c0069cd9db33cd3a7e39, store=B 2024-12-07T15:14:21,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:14:21,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b393b7989f53c0069cd9db33cd3a7e39, store=C 2024-12-07T15:14:21,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:14:21,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207056d1ade2a984ee2a09c328fd041a234_b393b7989f53c0069cd9db33cd3a7e39 is 50, key is test_row_0/A:col10/1733584460583/Put/seqid=0 2024-12-07T15:14:21,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742008_1184 (size=12454) 2024-12-07T15:14:21,745 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. as already flushing 2024-12-07T15:14:21,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on b393b7989f53c0069cd9db33cd3a7e39 2024-12-07T15:14:21,778 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733584461759, max timestamp=1733584461775Num uniques 859; 2024-12-07T15:14:21,779 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-07T15:14:21,779 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733584461759, max timestamp=1733584461775Num uniques 859; 2024-12-07T15:14:21,779 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733584461759, max timestamp=1733584461775Num uniques 859; 2024-12-07T15:14:21,779 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-07T15:14:21,779 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-07T15:14:21,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:21,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38038 deadline: 1733584521814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:21,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:21,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:21,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:21,828 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:21,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38006 deadline: 1733584521815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:21,829 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:21,829 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:21,829 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:21,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38036 deadline: 1733584521815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:21,829 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:21,830 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:21,830 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:21,830 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:21,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:21,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38038 deadline: 1733584521923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:21,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:21,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:21,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:21,937 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:21,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38006 deadline: 1733584521933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:21,937 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:21,937 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:21,937 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:21,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38036 deadline: 1733584521933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:21,937 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:21,938 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:21,938 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:21,938 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:22,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:22,023 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207056d1ade2a984ee2a09c328fd041a234_b393b7989f53c0069cd9db33cd3a7e39 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207056d1ade2a984ee2a09c328fd041a234_b393b7989f53c0069cd9db33cd3a7e39 2024-12-07T15:14:22,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/A/debab29f5ec241cca4a8e88c768e37b6, store: [table=TestAcidGuarantees family=A region=b393b7989f53c0069cd9db33cd3a7e39] 2024-12-07T15:14:22,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/A/debab29f5ec241cca4a8e88c768e37b6 is 175, key is test_row_0/A:col10/1733584460583/Put/seqid=0 2024-12-07T15:14:22,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742009_1185 (size=31255) 2024-12-07T15:14:22,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-07T15:14:22,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:22,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38038 deadline: 1733584522133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:22,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:22,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:22,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:22,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:22,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38006 deadline: 1733584522143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:22,146 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:22,146 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:22,146 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:22,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38036 deadline: 1733584522143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:22,146 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:22,146 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:22,146 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:22,146 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:22,430 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=273, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/A/debab29f5ec241cca4a8e88c768e37b6 2024-12-07T15:14:22,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/B/e3952bf2c768468a8364dcfab134a557 is 50, key is test_row_0/B:col10/1733584460583/Put/seqid=0 2024-12-07T15:14:22,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742010_1186 (size=12301) 2024-12-07T15:14:22,443 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/B/e3952bf2c768468a8364dcfab134a557 2024-12-07T15:14:22,444 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:22,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38038 deadline: 1733584522443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:22,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:22,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:22,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:22,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/C/733edc4ef2054a7b99597fc8e11c9a12 is 50, key is test_row_0/C:col10/1733584460583/Put/seqid=0 2024-12-07T15:14:22,454 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:22,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38006 deadline: 1733584522453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:22,454 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:22,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38036 deadline: 1733584522453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:22,455 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:22,455 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:22,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:22,455 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:22,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:22,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:22,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742011_1187 (size=12301) 2024-12-07T15:14:22,859 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/C/733edc4ef2054a7b99597fc8e11c9a12 2024-12-07T15:14:22,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/A/debab29f5ec241cca4a8e88c768e37b6 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/debab29f5ec241cca4a8e88c768e37b6 2024-12-07T15:14:22,873 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/debab29f5ec241cca4a8e88c768e37b6, entries=150, sequenceid=273, filesize=30.5 K 2024-12-07T15:14:22,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/B/e3952bf2c768468a8364dcfab134a557 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/e3952bf2c768468a8364dcfab134a557 2024-12-07T15:14:22,880 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/e3952bf2c768468a8364dcfab134a557, entries=150, sequenceid=273, filesize=12.0 K 2024-12-07T15:14:22,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/C/733edc4ef2054a7b99597fc8e11c9a12 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/733edc4ef2054a7b99597fc8e11c9a12 2024-12-07T15:14:22,887 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/733edc4ef2054a7b99597fc8e11c9a12, entries=150, sequenceid=273, filesize=12.0 K 2024-12-07T15:14:22,888 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for b393b7989f53c0069cd9db33cd3a7e39 in 1283ms, sequenceid=273, compaction requested=true 2024-12-07T15:14:22,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for b393b7989f53c0069cd9db33cd3a7e39: 2024-12-07T15:14:22,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:22,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-12-07T15:14:22,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-12-07T15:14:22,892 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-12-07T15:14:22,892 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9130 sec 2024-12-07T15:14:22,895 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.9190 sec 2024-12-07T15:14:22,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on b393b7989f53c0069cd9db33cd3a7e39 2024-12-07T15:14:22,965 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b393b7989f53c0069cd9db33cd3a7e39 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-12-07T15:14:22,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b393b7989f53c0069cd9db33cd3a7e39, store=A 2024-12-07T15:14:22,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T15:14:22,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b393b7989f53c0069cd9db33cd3a7e39, store=B 2024-12-07T15:14:22,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T15:14:22,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b393b7989f53c0069cd9db33cd3a7e39, store=C 2024-12-07T15:14:22,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T15:14:22,977 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412071835c78ba4904e32af9606a2cb39c1be_b393b7989f53c0069cd9db33cd3a7e39 is 50, key is test_row_0/A:col10/1733584461813/Put/seqid=0 2024-12-07T15:14:22,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:22,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38006 deadline: 1733584522976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:22,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:22,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38036 deadline: 1733584522977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:22,983 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:22,983 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:22,983 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:22,983 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:22,983 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:22,983 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:22,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:22,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38038 deadline: 1733584522979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:22,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:22,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:22,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:22,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742012_1188 (size=14994) 2024-12-07T15:14:22,994 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:22,999 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412071835c78ba4904e32af9606a2cb39c1be_b393b7989f53c0069cd9db33cd3a7e39 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412071835c78ba4904e32af9606a2cb39c1be_b393b7989f53c0069cd9db33cd3a7e39 2024-12-07T15:14:23,000 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/A/e1e4c4770e454a699b1778fb17db18ff, store: [table=TestAcidGuarantees family=A region=b393b7989f53c0069cd9db33cd3a7e39] 2024-12-07T15:14:23,001 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/A/e1e4c4770e454a699b1778fb17db18ff is 175, key is test_row_0/A:col10/1733584461813/Put/seqid=0 2024-12-07T15:14:23,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742013_1189 (size=39949) 2024-12-07T15:14:23,094 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:23,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38036 deadline: 1733584523093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:23,095 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:23,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38006 deadline: 1733584523093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:23,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:23,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:23,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:23,095 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:23,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38038 deadline: 1733584523094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:23,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:23,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:23,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:23,097 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:23,098 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:23,098 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:23,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-07T15:14:23,133 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T15:14:23,134 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T15:14:23,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-12-07T15:14:23,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-07T15:14:23,137 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T15:14:23,138 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T15:14:23,138 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T15:14:23,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-07T15:14:23,291 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-07T15:14:23,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:23,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. as already flushing 2024-12-07T15:14:23,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:23,292 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:23,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:23,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:23,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:23,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38036 deadline: 1733584523303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:23,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:23,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:23,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:23,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38006 deadline: 1733584523303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:23,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:23,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:23,307 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:23,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38038 deadline: 1733584523303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:23,307 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:23,307 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:23,308 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:23,308 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:23,308 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:23,424 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=310, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/A/e1e4c4770e454a699b1778fb17db18ff 2024-12-07T15:14:23,433 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/B/38b157532f8a44de9dc61881a2f2fd80 is 50, key is test_row_0/B:col10/1733584461813/Put/seqid=0 2024-12-07T15:14:23,445 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-07T15:14:23,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:23,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. as already flushing 2024-12-07T15:14:23,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:23,446 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:23,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:23,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:23,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-07T15:14:23,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742014_1190 (size=12301) 2024-12-07T15:14:23,463 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/B/38b157532f8a44de9dc61881a2f2fd80 2024-12-07T15:14:23,477 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/C/b515dcbfe9024e0d9a3c73f1b5d8d8f8 is 50, key is test_row_0/C:col10/1733584461813/Put/seqid=0 2024-12-07T15:14:23,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742015_1191 (size=12301) 2024-12-07T15:14:23,495 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/C/b515dcbfe9024e0d9a3c73f1b5d8d8f8 2024-12-07T15:14:23,509 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/A/e1e4c4770e454a699b1778fb17db18ff as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/e1e4c4770e454a699b1778fb17db18ff 2024-12-07T15:14:23,516 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/e1e4c4770e454a699b1778fb17db18ff, entries=200, sequenceid=310, filesize=39.0 K 2024-12-07T15:14:23,518 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/B/38b157532f8a44de9dc61881a2f2fd80 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/38b157532f8a44de9dc61881a2f2fd80 2024-12-07T15:14:23,526 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/38b157532f8a44de9dc61881a2f2fd80, entries=150, sequenceid=310, filesize=12.0 K 2024-12-07T15:14:23,527 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/C/b515dcbfe9024e0d9a3c73f1b5d8d8f8 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/b515dcbfe9024e0d9a3c73f1b5d8d8f8 2024-12-07T15:14:23,546 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/b515dcbfe9024e0d9a3c73f1b5d8d8f8, entries=150, sequenceid=310, filesize=12.0 K 2024-12-07T15:14:23,547 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for b393b7989f53c0069cd9db33cd3a7e39 in 582ms, sequenceid=310, compaction requested=true 2024-12-07T15:14:23,548 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b393b7989f53c0069cd9db33cd3a7e39: 2024-12-07T15:14:23,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b393b7989f53c0069cd9db33cd3a7e39:A, priority=-2147483648, current under compaction store size is 1 2024-12-07T15:14:23,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:14:23,548 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-07T15:14:23,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b393b7989f53c0069cd9db33cd3a7e39:B, priority=-2147483648, current under compaction store size is 2 2024-12-07T15:14:23,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:14:23,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b393b7989f53c0069cd9db33cd3a7e39:C, priority=-2147483648, current under compaction store size is 3 2024-12-07T15:14:23,548 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-07T15:14:23,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T15:14:23,550 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142636 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-07T15:14:23,550 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1541): b393b7989f53c0069cd9db33cd3a7e39/A is initiating minor compaction (all files) 2024-12-07T15:14:23,550 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49432 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-07T15:14:23,550 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b393b7989f53c0069cd9db33cd3a7e39/A in TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:23,550 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1541): b393b7989f53c0069cd9db33cd3a7e39/B is initiating minor compaction (all files) 2024-12-07T15:14:23,550 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b393b7989f53c0069cd9db33cd3a7e39/B in TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:23,550 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/1c61771a38004224bd44cf2faa552483, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/73d8a1eff7a34169ad538f47feb9405e, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/debab29f5ec241cca4a8e88c768e37b6, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/e1e4c4770e454a699b1778fb17db18ff] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp, totalSize=139.3 K 2024-12-07T15:14:23,550 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:23,550 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/e000b6d727814135aa3c6b38b2758961, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/6e18442b5c234b2397e183e20d5aff63, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/e3952bf2c768468a8364dcfab134a557, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/38b157532f8a44de9dc61881a2f2fd80] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp, totalSize=48.3 K 2024-12-07T15:14:23,550 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. files: [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/1c61771a38004224bd44cf2faa552483, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/73d8a1eff7a34169ad538f47feb9405e, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/debab29f5ec241cca4a8e88c768e37b6, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/e1e4c4770e454a699b1778fb17db18ff] 2024-12-07T15:14:23,551 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting e000b6d727814135aa3c6b38b2758961, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1733584458248 2024-12-07T15:14:23,551 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c61771a38004224bd44cf2faa552483, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1733584458248 2024-12-07T15:14:23,551 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 73d8a1eff7a34169ad538f47feb9405e, keycount=200, bloomtype=ROW, size=38.9 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1733584459423 2024-12-07T15:14:23,552 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e18442b5c234b2397e183e20d5aff63, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1733584459423 2024-12-07T15:14:23,554 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting debab29f5ec241cca4a8e88c768e37b6, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1733584460583 2024-12-07T15:14:23,554 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting e3952bf2c768468a8364dcfab134a557, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1733584460583 2024-12-07T15:14:23,554 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting e1e4c4770e454a699b1778fb17db18ff, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1733584461785 2024-12-07T15:14:23,555 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 38b157532f8a44de9dc61881a2f2fd80, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1733584461785 2024-12-07T15:14:23,567 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b393b7989f53c0069cd9db33cd3a7e39] 2024-12-07T15:14:23,591 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b393b7989f53c0069cd9db33cd3a7e39#B#compaction#159 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:14:23,592 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/B/7140895f2ce441f98d14e666b2e58e47 is 50, key is test_row_0/B:col10/1733584461813/Put/seqid=0 2024-12-07T15:14:23,599 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-07T15:14:23,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:23,599 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing b393b7989f53c0069cd9db33cd3a7e39 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-12-07T15:14:23,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b393b7989f53c0069cd9db33cd3a7e39, store=A 2024-12-07T15:14:23,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:14:23,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b393b7989f53c0069cd9db33cd3a7e39, store=B 2024-12-07T15:14:23,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:14:23,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b393b7989f53c0069cd9db33cd3a7e39, store=C 2024-12-07T15:14:23,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:14:23,609 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241207065f015f81d54af492dff5bd5b822a90_b393b7989f53c0069cd9db33cd3a7e39 store=[table=TestAcidGuarantees family=A region=b393b7989f53c0069cd9db33cd3a7e39] 2024-12-07T15:14:23,612 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241207065f015f81d54af492dff5bd5b822a90_b393b7989f53c0069cd9db33cd3a7e39, store=[table=TestAcidGuarantees family=A region=b393b7989f53c0069cd9db33cd3a7e39] 2024-12-07T15:14:23,612 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207065f015f81d54af492dff5bd5b822a90_b393b7989f53c0069cd9db33cd3a7e39 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b393b7989f53c0069cd9db33cd3a7e39] 2024-12-07T15:14:23,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on b393b7989f53c0069cd9db33cd3a7e39 2024-12-07T15:14:23,628 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. as already flushing 2024-12-07T15:14:23,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742017_1193 (size=4469) 2024-12-07T15:14:23,656 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b393b7989f53c0069cd9db33cd3a7e39#A#compaction#158 average throughput is 0.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:14:23,656 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/A/7a4c75f2a9aa49c1a32f59f246603249 is 175, key is test_row_0/A:col10/1733584461813/Put/seqid=0 2024-12-07T15:14:23,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742016_1192 (size=12915) 2024-12-07T15:14:23,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207a55a25b7421e4f449d1a2f15094baefd_b393b7989f53c0069cd9db33cd3a7e39 is 50, key is test_row_0/A:col10/1733584462977/Put/seqid=0 2024-12-07T15:14:23,696 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733584463651, max timestamp=1733584463692Num uniques 859; 2024-12-07T15:14:23,696 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-07T15:14:23,699 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733584463651, max timestamp=1733584463692Num uniques 859; 2024-12-07T15:14:23,699 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-07T15:14:23,702 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733584463651, max timestamp=1733584463692Num uniques 859; 2024-12-07T15:14:23,702 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-07T15:14:23,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742018_1194 (size=31869) 2024-12-07T15:14:23,724 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/A/7a4c75f2a9aa49c1a32f59f246603249 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/7a4c75f2a9aa49c1a32f59f246603249 2024-12-07T15:14:23,732 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b393b7989f53c0069cd9db33cd3a7e39/A of b393b7989f53c0069cd9db33cd3a7e39 into 7a4c75f2a9aa49c1a32f59f246603249(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:14:23,732 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b393b7989f53c0069cd9db33cd3a7e39: 2024-12-07T15:14:23,732 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., storeName=b393b7989f53c0069cd9db33cd3a7e39/A, priority=12, startTime=1733584463548; duration=0sec 2024-12-07T15:14:23,733 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T15:14:23,733 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b393b7989f53c0069cd9db33cd3a7e39:A 2024-12-07T15:14:23,733 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-07T15:14:23,735 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49432 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-07T15:14:23,735 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1541): b393b7989f53c0069cd9db33cd3a7e39/C is initiating minor compaction (all files) 2024-12-07T15:14:23,735 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b393b7989f53c0069cd9db33cd3a7e39/C in TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:23,735 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/96f8121fa74b460eb6249314f02d8c6b, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/5650c4e397e245c0aff67bca28e594e9, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/733edc4ef2054a7b99597fc8e11c9a12, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/b515dcbfe9024e0d9a3c73f1b5d8d8f8] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp, totalSize=48.3 K 2024-12-07T15:14:23,736 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 96f8121fa74b460eb6249314f02d8c6b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1733584458248 2024-12-07T15:14:23,736 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5650c4e397e245c0aff67bca28e594e9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1733584459423 2024-12-07T15:14:23,737 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 733edc4ef2054a7b99597fc8e11c9a12, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1733584460583 2024-12-07T15:14:23,737 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting b515dcbfe9024e0d9a3c73f1b5d8d8f8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1733584461785 2024-12-07T15:14:23,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742019_1195 (size=12454) 2024-12-07T15:14:23,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:23,757 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207a55a25b7421e4f449d1a2f15094baefd_b393b7989f53c0069cd9db33cd3a7e39 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207a55a25b7421e4f449d1a2f15094baefd_b393b7989f53c0069cd9db33cd3a7e39 2024-12-07T15:14:23,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/A/68b5c8fce8e84b1f932367853b81adf2, store: [table=TestAcidGuarantees family=A region=b393b7989f53c0069cd9db33cd3a7e39] 2024-12-07T15:14:23,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/A/68b5c8fce8e84b1f932367853b81adf2 is 175, key is test_row_0/A:col10/1733584462977/Put/seqid=0 2024-12-07T15:14:23,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-07T15:14:23,767 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b393b7989f53c0069cd9db33cd3a7e39#C#compaction#161 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:14:23,768 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/C/79b408ba5bca402b89bc567337bc014f is 50, key is test_row_0/C:col10/1733584461813/Put/seqid=0 2024-12-07T15:14:23,780 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:23,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38036 deadline: 1733584523773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:23,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:23,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:23,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:23,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:23,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38038 deadline: 1733584523775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:23,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:23,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:23,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:23,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:23,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38006 deadline: 1733584523776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:23,784 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:23,784 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:23,784 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:23,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742020_1196 (size=31255) 2024-12-07T15:14:23,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742021_1197 (size=12915) 2024-12-07T15:14:23,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:23,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38036 deadline: 1733584523883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:23,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:23,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:23,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:23,907 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:23,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38038 deadline: 1733584523893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:23,908 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:23,908 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:23,908 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:23,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:23,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38006 deadline: 1733584523893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:23,908 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:23,908 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:23,908 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:24,085 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/B/7140895f2ce441f98d14e666b2e58e47 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/7140895f2ce441f98d14e666b2e58e47 2024-12-07T15:14:24,091 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b393b7989f53c0069cd9db33cd3a7e39/B of b393b7989f53c0069cd9db33cd3a7e39 into 7140895f2ce441f98d14e666b2e58e47(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:14:24,091 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b393b7989f53c0069cd9db33cd3a7e39: 2024-12-07T15:14:24,091 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., storeName=b393b7989f53c0069cd9db33cd3a7e39/B, priority=12, startTime=1733584463548; duration=0sec 2024-12-07T15:14:24,091 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:14:24,091 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b393b7989f53c0069cd9db33cd3a7e39:B 2024-12-07T15:14:24,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:24,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38036 deadline: 1733584524094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:24,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:24,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:24,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:24,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:24,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38038 deadline: 1733584524113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:24,119 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:24,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38006 deadline: 1733584524113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:24,119 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:24,119 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:24,119 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:24,121 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:24,121 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:24,121 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:24,190 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=319, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/A/68b5c8fce8e84b1f932367853b81adf2 2024-12-07T15:14:24,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/B/be6f890b87fb41cc9f0b5db4b56cb9be is 50, key is test_row_0/B:col10/1733584462977/Put/seqid=0 2024-12-07T15:14:24,223 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/C/79b408ba5bca402b89bc567337bc014f as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/79b408ba5bca402b89bc567337bc014f 2024-12-07T15:14:24,230 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b393b7989f53c0069cd9db33cd3a7e39/C of b393b7989f53c0069cd9db33cd3a7e39 into 79b408ba5bca402b89bc567337bc014f(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:14:24,230 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b393b7989f53c0069cd9db33cd3a7e39: 2024-12-07T15:14:24,231 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., storeName=b393b7989f53c0069cd9db33cd3a7e39/C, priority=12, startTime=1733584463548; duration=0sec 2024-12-07T15:14:24,231 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:14:24,231 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b393b7989f53c0069cd9db33cd3a7e39:C 2024-12-07T15:14:24,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742022_1198 (size=12301) 2024-12-07T15:14:24,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-07T15:14:24,298 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:24,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37972 deadline: 1733584524293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:24,298 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:24,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38020 deadline: 1733584524293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:24,298 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to d6797ebe9c52,46009,1733584418401 for 'test_row_1' in b393b7989f53c0069cd9db33cd3a7e39 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8218 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-07T15:14:24,299 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:24,299 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:24,299 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:24,299 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to d6797ebe9c52,46009,1733584418401 for 'test_row_2' in b393b7989f53c0069cd9db33cd3a7e39 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8217 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-07T15:14:24,299 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:24,300 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:24,300 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:24,422 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:24,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38036 deadline: 1733584524414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:24,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:24,425 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:24,425 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:24,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:24,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38006 deadline: 1733584524424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:24,426 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:24,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:24,426 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:24,426 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:24,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38038 deadline: 1733584524424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:24,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:24,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:24,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:24,655 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/B/be6f890b87fb41cc9f0b5db4b56cb9be 2024-12-07T15:14:24,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/C/a575fb48387f425b95ef505b2a794cb7 is 50, key is test_row_0/C:col10/1733584462977/Put/seqid=0 2024-12-07T15:14:24,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742023_1199 (size=12301) 2024-12-07T15:14:24,730 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/C/a575fb48387f425b95ef505b2a794cb7 2024-12-07T15:14:24,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/A/68b5c8fce8e84b1f932367853b81adf2 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/68b5c8fce8e84b1f932367853b81adf2 2024-12-07T15:14:24,747 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/68b5c8fce8e84b1f932367853b81adf2, entries=150, sequenceid=319, filesize=30.5 K 2024-12-07T15:14:24,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/B/be6f890b87fb41cc9f0b5db4b56cb9be as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/be6f890b87fb41cc9f0b5db4b56cb9be 2024-12-07T15:14:24,757 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/be6f890b87fb41cc9f0b5db4b56cb9be, entries=150, sequenceid=319, filesize=12.0 K 2024-12-07T15:14:24,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/C/a575fb48387f425b95ef505b2a794cb7 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/a575fb48387f425b95ef505b2a794cb7 2024-12-07T15:14:24,765 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/a575fb48387f425b95ef505b2a794cb7, entries=150, sequenceid=319, filesize=12.0 K 2024-12-07T15:14:24,770 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=143.13 KB/146560 for b393b7989f53c0069cd9db33cd3a7e39 in 1171ms, sequenceid=319, compaction requested=false 2024-12-07T15:14:24,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for b393b7989f53c0069cd9db33cd3a7e39: 2024-12-07T15:14:24,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:24,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-12-07T15:14:24,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-12-07T15:14:24,774 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-12-07T15:14:24,774 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6340 sec 2024-12-07T15:14:24,782 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.6400 sec 2024-12-07T15:14:24,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on b393b7989f53c0069cd9db33cd3a7e39 2024-12-07T15:14:24,938 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b393b7989f53c0069cd9db33cd3a7e39 3/3 column families, dataSize=149.83 KB heapSize=420.53 KB 2024-12-07T15:14:24,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b393b7989f53c0069cd9db33cd3a7e39, store=A 2024-12-07T15:14:24,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T15:14:24,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b393b7989f53c0069cd9db33cd3a7e39, store=B 2024-12-07T15:14:24,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T15:14:24,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b393b7989f53c0069cd9db33cd3a7e39, store=C 2024-12-07T15:14:24,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T15:14:24,950 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207fd586cc427594221be0e4ac160ff1f41_b393b7989f53c0069cd9db33cd3a7e39 is 50, key is test_row_0/A:col10/1733584463772/Put/seqid=0 2024-12-07T15:14:24,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:24,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38038 deadline: 1733584524951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:24,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:24,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:24,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:24,958 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:24,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38006 deadline: 1733584524952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:24,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742024_1200 (size=14994) 2024-12-07T15:14:24,959 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:24,959 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:24,959 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:24,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:24,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38036 deadline: 1733584524957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:24,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:24,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:24,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:25,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:25,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38038 deadline: 1733584525064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:25,068 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:25,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:25,069 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:25,069 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:25,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38006 deadline: 1733584525064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:25,069 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:25,069 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:25,069 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:25,075 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:25,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38036 deadline: 1733584525073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:25,076 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:25,076 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:25,076 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:25,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:25,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38036 deadline: 1733584525284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:25,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:25,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:25,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38038 deadline: 1733584525284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:25,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:25,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:25,291 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:25,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38006 deadline: 1733584525284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:25,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:25,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:25,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:25,292 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:25,292 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:25,292 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:25,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-07T15:14:25,293 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T15:14:25,294 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T15:14:25,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-12-07T15:14:25,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-07T15:14:25,296 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T15:14:25,297 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T15:14:25,297 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T15:14:25,359 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:25,364 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207fd586cc427594221be0e4ac160ff1f41_b393b7989f53c0069cd9db33cd3a7e39 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207fd586cc427594221be0e4ac160ff1f41_b393b7989f53c0069cd9db33cd3a7e39 2024-12-07T15:14:25,365 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/A/9668f3d9ebbd4224b094cf8282421e6a, store: [table=TestAcidGuarantees family=A region=b393b7989f53c0069cd9db33cd3a7e39] 2024-12-07T15:14:25,366 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/A/9668f3d9ebbd4224b094cf8282421e6a is 175, key is test_row_0/A:col10/1733584463772/Put/seqid=0 2024-12-07T15:14:25,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742025_1201 (size=39949) 2024-12-07T15:14:25,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-07T15:14:25,449 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-07T15:14:25,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:25,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. as already flushing 2024-12-07T15:14:25,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:25,449 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:25,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:25,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:25,603 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-07T15:14:25,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:25,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. as already flushing 2024-12-07T15:14:25,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:25,604 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:25,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:25,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:25,609 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:25,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38036 deadline: 1733584525604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:25,609 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:25,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38038 deadline: 1733584525604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:25,609 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:25,610 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:25,610 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:25,610 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:25,610 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:25,610 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:25,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38006 deadline: 1733584525604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:25,610 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:25,610 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:25,610 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:25,610 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:25,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-07T15:14:25,756 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-07T15:14:25,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:25,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. as already flushing 2024-12-07T15:14:25,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:25,757 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:25,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:25,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:25,775 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=359, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/A/9668f3d9ebbd4224b094cf8282421e6a 2024-12-07T15:14:25,787 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/B/0786bd92ae8e4c1085ccca93df4c68d4 is 50, key is test_row_0/B:col10/1733584463772/Put/seqid=0 2024-12-07T15:14:25,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742026_1202 (size=12301) 2024-12-07T15:14:25,794 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/B/0786bd92ae8e4c1085ccca93df4c68d4 2024-12-07T15:14:25,818 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/C/e1ae27d46caa46e0aa05fa2a7308b4e8 is 50, key is test_row_0/C:col10/1733584463772/Put/seqid=0 2024-12-07T15:14:25,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742027_1203 (size=12301) 2024-12-07T15:14:25,834 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/C/e1ae27d46caa46e0aa05fa2a7308b4e8 2024-12-07T15:14:25,841 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/A/9668f3d9ebbd4224b094cf8282421e6a as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/9668f3d9ebbd4224b094cf8282421e6a 2024-12-07T15:14:25,845 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/9668f3d9ebbd4224b094cf8282421e6a, entries=200, sequenceid=359, filesize=39.0 K 2024-12-07T15:14:25,846 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/B/0786bd92ae8e4c1085ccca93df4c68d4 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/0786bd92ae8e4c1085ccca93df4c68d4 2024-12-07T15:14:25,851 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/0786bd92ae8e4c1085ccca93df4c68d4, entries=150, sequenceid=359, filesize=12.0 K 2024-12-07T15:14:25,853 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/C/e1ae27d46caa46e0aa05fa2a7308b4e8 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/e1ae27d46caa46e0aa05fa2a7308b4e8 2024-12-07T15:14:25,859 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/e1ae27d46caa46e0aa05fa2a7308b4e8, entries=150, sequenceid=359, filesize=12.0 K 2024-12-07T15:14:25,860 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~149.83 KB/153430, heapSize ~420.48 KB/430576, currentSize=33.54 KB/34350 for b393b7989f53c0069cd9db33cd3a7e39 in 922ms, sequenceid=359, compaction requested=true 2024-12-07T15:14:25,860 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b393b7989f53c0069cd9db33cd3a7e39: 2024-12-07T15:14:25,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b393b7989f53c0069cd9db33cd3a7e39:A, priority=-2147483648, current under compaction store size is 1 2024-12-07T15:14:25,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:14:25,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b393b7989f53c0069cd9db33cd3a7e39:B, priority=-2147483648, current under compaction store size is 2 2024-12-07T15:14:25,860 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:14:25,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:14:25,860 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:14:25,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b393b7989f53c0069cd9db33cd3a7e39:C, priority=-2147483648, current under compaction store size is 3 2024-12-07T15:14:25,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T15:14:25,862 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:14:25,862 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1541): b393b7989f53c0069cd9db33cd3a7e39/B is initiating minor compaction (all files) 2024-12-07T15:14:25,862 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b393b7989f53c0069cd9db33cd3a7e39/B in TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:25,862 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/7140895f2ce441f98d14e666b2e58e47, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/be6f890b87fb41cc9f0b5db4b56cb9be, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/0786bd92ae8e4c1085ccca93df4c68d4] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp, totalSize=36.6 K 2024-12-07T15:14:25,862 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103073 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:14:25,862 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1541): b393b7989f53c0069cd9db33cd3a7e39/A is initiating minor compaction (all files) 2024-12-07T15:14:25,862 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b393b7989f53c0069cd9db33cd3a7e39/A in TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:25,862 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/7a4c75f2a9aa49c1a32f59f246603249, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/68b5c8fce8e84b1f932367853b81adf2, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/9668f3d9ebbd4224b094cf8282421e6a] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp, totalSize=100.7 K 2024-12-07T15:14:25,862 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 7140895f2ce441f98d14e666b2e58e47, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1733584461785 2024-12-07T15:14:25,862 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:25,862 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. files: [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/7a4c75f2a9aa49c1a32f59f246603249, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/68b5c8fce8e84b1f932367853b81adf2, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/9668f3d9ebbd4224b094cf8282421e6a] 2024-12-07T15:14:25,863 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting be6f890b87fb41cc9f0b5db4b56cb9be, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1733584462967 2024-12-07T15:14:25,863 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a4c75f2a9aa49c1a32f59f246603249, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1733584461785 2024-12-07T15:14:25,863 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 0786bd92ae8e4c1085ccca93df4c68d4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1733584463772 2024-12-07T15:14:25,864 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 68b5c8fce8e84b1f932367853b81adf2, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1733584462967 2024-12-07T15:14:25,864 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9668f3d9ebbd4224b094cf8282421e6a, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1733584463769 2024-12-07T15:14:25,892 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b393b7989f53c0069cd9db33cd3a7e39#B#compaction#167 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:14:25,893 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/B/9a45344881274d838e2d0a8dd8613e9b is 50, key is test_row_0/B:col10/1733584463772/Put/seqid=0 2024-12-07T15:14:25,895 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b393b7989f53c0069cd9db33cd3a7e39] 2024-12-07T15:14:25,901 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241207fcbec626ad1d441a84affc0e48bc5c45_b393b7989f53c0069cd9db33cd3a7e39 store=[table=TestAcidGuarantees family=A region=b393b7989f53c0069cd9db33cd3a7e39] 2024-12-07T15:14:25,903 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241207fcbec626ad1d441a84affc0e48bc5c45_b393b7989f53c0069cd9db33cd3a7e39, store=[table=TestAcidGuarantees family=A region=b393b7989f53c0069cd9db33cd3a7e39] 2024-12-07T15:14:25,903 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207fcbec626ad1d441a84affc0e48bc5c45_b393b7989f53c0069cd9db33cd3a7e39 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b393b7989f53c0069cd9db33cd3a7e39] 2024-12-07T15:14:25,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742028_1204 (size=13017) 2024-12-07T15:14:25,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742029_1205 (size=4469) 2024-12-07T15:14:25,910 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-07T15:14:25,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:25,911 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing b393b7989f53c0069cd9db33cd3a7e39 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-07T15:14:25,911 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b393b7989f53c0069cd9db33cd3a7e39#A#compaction#168 average throughput is 1.53 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:14:25,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b393b7989f53c0069cd9db33cd3a7e39, store=A 2024-12-07T15:14:25,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:14:25,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b393b7989f53c0069cd9db33cd3a7e39, store=B 2024-12-07T15:14:25,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:14:25,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b393b7989f53c0069cd9db33cd3a7e39, store=C 2024-12-07T15:14:25,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:14:25,912 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/A/d3e9422e0a64435490077642cba5ba61 is 175, key is test_row_0/A:col10/1733584463772/Put/seqid=0 2024-12-07T15:14:25,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-07T15:14:25,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207ca766206f1e9469882f402b2fb03228e_b393b7989f53c0069cd9db33cd3a7e39 is 50, key is test_row_0/A:col10/1733584464948/Put/seqid=0 2024-12-07T15:14:25,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742030_1206 (size=31971) 2024-12-07T15:14:25,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742031_1207 (size=12454) 2024-12-07T15:14:26,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on b393b7989f53c0069cd9db33cd3a7e39 2024-12-07T15:14:26,126 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. as already flushing 2024-12-07T15:14:26,177 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733584466145, max timestamp=1733584466173Num uniques 859; 2024-12-07T15:14:26,177 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733584466145, max timestamp=1733584466173Num uniques 859; 2024-12-07T15:14:26,177 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-07T15:14:26,177 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-07T15:14:26,178 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733584466145, max timestamp=1733584466173Num uniques 859; 2024-12-07T15:14:26,178 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-07T15:14:26,249 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:26,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38038 deadline: 1733584526243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:26,250 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:26,250 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:26,250 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:26,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:26,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38006 deadline: 1733584526250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:26,255 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733584466234, max timestamp=1733584466250Num uniques 1118; 2024-12-07T15:14:26,255 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=7688, suffix heap size=144240,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=168 2024-12-07T15:14:26,255 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:26,255 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:26,255 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:26,257 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733584466234, max timestamp=1733584466250Num uniques 1118; 2024-12-07T15:14:26,258 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=7688, suffix heap size=144240,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=168 2024-12-07T15:14:26,258 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733584466234, max timestamp=1733584466250Num uniques 1118; 2024-12-07T15:14:26,258 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=7688, suffix heap size=144240,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=168 2024-12-07T15:14:26,314 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/B/9a45344881274d838e2d0a8dd8613e9b as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/9a45344881274d838e2d0a8dd8613e9b 2024-12-07T15:14:26,320 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b393b7989f53c0069cd9db33cd3a7e39/B of b393b7989f53c0069cd9db33cd3a7e39 into 9a45344881274d838e2d0a8dd8613e9b(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:14:26,320 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b393b7989f53c0069cd9db33cd3a7e39: 2024-12-07T15:14:26,320 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., storeName=b393b7989f53c0069cd9db33cd3a7e39/B, priority=13, startTime=1733584465860; duration=0sec 2024-12-07T15:14:26,321 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T15:14:26,321 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b393b7989f53c0069cd9db33cd3a7e39:B 2024-12-07T15:14:26,321 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:14:26,322 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:14:26,327 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1541): b393b7989f53c0069cd9db33cd3a7e39/C is initiating minor compaction (all files) 2024-12-07T15:14:26,327 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b393b7989f53c0069cd9db33cd3a7e39/C in TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:26,327 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/79b408ba5bca402b89bc567337bc014f, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/a575fb48387f425b95ef505b2a794cb7, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/e1ae27d46caa46e0aa05fa2a7308b4e8] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp, totalSize=36.6 K 2024-12-07T15:14:26,327 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 79b408ba5bca402b89bc567337bc014f, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1733584461785 2024-12-07T15:14:26,328 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting a575fb48387f425b95ef505b2a794cb7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1733584462967 2024-12-07T15:14:26,329 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting e1ae27d46caa46e0aa05fa2a7308b4e8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1733584463772 2024-12-07T15:14:26,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:26,334 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/A/d3e9422e0a64435490077642cba5ba61 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/d3e9422e0a64435490077642cba5ba61 2024-12-07T15:14:26,337 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207ca766206f1e9469882f402b2fb03228e_b393b7989f53c0069cd9db33cd3a7e39 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207ca766206f1e9469882f402b2fb03228e_b393b7989f53c0069cd9db33cd3a7e39 2024-12-07T15:14:26,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/A/22b3b6c20eed4a3e9ca1878796f5709b, store: [table=TestAcidGuarantees family=A region=b393b7989f53c0069cd9db33cd3a7e39] 2024-12-07T15:14:26,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/A/22b3b6c20eed4a3e9ca1878796f5709b is 175, key is test_row_0/A:col10/1733584464948/Put/seqid=0 2024-12-07T15:14:26,345 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b393b7989f53c0069cd9db33cd3a7e39#C#compaction#170 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:14:26,345 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/C/7cab36cf861b4b2a9517682255982d42 is 50, key is test_row_0/C:col10/1733584463772/Put/seqid=0 2024-12-07T15:14:26,349 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b393b7989f53c0069cd9db33cd3a7e39/A of b393b7989f53c0069cd9db33cd3a7e39 into d3e9422e0a64435490077642cba5ba61(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:14:26,349 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b393b7989f53c0069cd9db33cd3a7e39: 2024-12-07T15:14:26,349 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., storeName=b393b7989f53c0069cd9db33cd3a7e39/A, priority=13, startTime=1733584465860; duration=0sec 2024-12-07T15:14:26,349 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:14:26,349 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b393b7989f53c0069cd9db33cd3a7e39:A 2024-12-07T15:14:26,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742032_1208 (size=31255) 2024-12-07T15:14:26,358 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=367, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/A/22b3b6c20eed4a3e9ca1878796f5709b 2024-12-07T15:14:26,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742033_1209 (size=13017) 2024-12-07T15:14:26,371 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=327, cellSize=14972, totalHeapSize=34248, min timestamp=1733584466310, max timestamp=1733584466367Num uniques 1027; 2024-12-07T15:14:26,372 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=14972, suffix heap size=134776,new segment heap size=34248 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=327 2024-12-07T15:14:26,372 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=327, cellSize=14972, totalHeapSize=34248, min timestamp=1733584466310, max timestamp=1733584466367Num uniques 1027; 2024-12-07T15:14:26,372 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=14972, suffix heap size=134776,new segment heap size=34248 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=327 2024-12-07T15:14:26,375 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733584466341, max timestamp=1733584466367Num uniques 1027; 2024-12-07T15:14:26,375 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-12-07T15:14:26,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/B/e770910ee8534813ae759076573b7e58 is 50, key is test_row_0/B:col10/1733584464948/Put/seqid=0 2024-12-07T15:14:26,390 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/C/7cab36cf861b4b2a9517682255982d42 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/7cab36cf861b4b2a9517682255982d42 2024-12-07T15:14:26,404 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b393b7989f53c0069cd9db33cd3a7e39/C of b393b7989f53c0069cd9db33cd3a7e39 into 7cab36cf861b4b2a9517682255982d42(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:14:26,404 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b393b7989f53c0069cd9db33cd3a7e39: 2024-12-07T15:14:26,404 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., storeName=b393b7989f53c0069cd9db33cd3a7e39/C, priority=13, startTime=1733584465860; duration=0sec 2024-12-07T15:14:26,405 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:14:26,405 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b393b7989f53c0069cd9db33cd3a7e39:C 2024-12-07T15:14:26,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742034_1210 (size=12301) 2024-12-07T15:14:26,408 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/B/e770910ee8534813ae759076573b7e58 2024-12-07T15:14:26,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/C/7d83597e3dfa455fb1d954406fc035dc is 50, key is test_row_0/C:col10/1733584464948/Put/seqid=0 2024-12-07T15:14:26,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:26,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38006 deadline: 1733584526417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:26,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:26,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-07T15:14:26,435 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:26,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38038 deadline: 1733584526433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:26,435 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:26,435 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:26,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:26,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:26,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:26,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742035_1211 (size=12301) 2024-12-07T15:14:26,440 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/C/7d83597e3dfa455fb1d954406fc035dc 2024-12-07T15:14:26,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:26,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38036 deadline: 1733584526435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:26,441 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:26,441 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:26,441 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:26,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/A/22b3b6c20eed4a3e9ca1878796f5709b as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/22b3b6c20eed4a3e9ca1878796f5709b 2024-12-07T15:14:26,452 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/22b3b6c20eed4a3e9ca1878796f5709b, entries=150, sequenceid=367, filesize=30.5 K 2024-12-07T15:14:26,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/B/e770910ee8534813ae759076573b7e58 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/e770910ee8534813ae759076573b7e58 2024-12-07T15:14:26,495 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/e770910ee8534813ae759076573b7e58, entries=150, sequenceid=367, filesize=12.0 K 2024-12-07T15:14:26,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/C/7d83597e3dfa455fb1d954406fc035dc as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/7d83597e3dfa455fb1d954406fc035dc 2024-12-07T15:14:26,505 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/7d83597e3dfa455fb1d954406fc035dc, entries=150, sequenceid=367, filesize=12.0 K 2024-12-07T15:14:26,506 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=149.83 KB/153430 for b393b7989f53c0069cd9db33cd3a7e39 in 595ms, sequenceid=367, compaction requested=false 2024-12-07T15:14:26,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for b393b7989f53c0069cd9db33cd3a7e39: 2024-12-07T15:14:26,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:26,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-12-07T15:14:26,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-12-07T15:14:26,511 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-12-07T15:14:26,511 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2110 sec 2024-12-07T15:14:26,513 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 1.2180 sec 2024-12-07T15:14:26,549 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b393b7989f53c0069cd9db33cd3a7e39 3/3 column families, dataSize=156.54 KB heapSize=439.47 KB 2024-12-07T15:14:26,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b393b7989f53c0069cd9db33cd3a7e39, store=A 2024-12-07T15:14:26,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-07T15:14:26,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b393b7989f53c0069cd9db33cd3a7e39, store=B 2024-12-07T15:14:26,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-07T15:14:26,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b393b7989f53c0069cd9db33cd3a7e39, store=C 2024-12-07T15:14:26,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-07T15:14:26,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on b393b7989f53c0069cd9db33cd3a7e39 2024-12-07T15:14:26,573 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:26,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38038 deadline: 1733584526566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:26,573 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:26,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38036 deadline: 1733584526566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:26,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:26,574 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:26,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:26,574 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:26,574 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:26,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:26,574 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:26,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38006 deadline: 1733584526567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:26,575 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:26,575 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:26,575 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:26,588 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120719364c1dcb064babb3b73d6fa722f533_b393b7989f53c0069cd9db33cd3a7e39 is 50, key is test_row_0/A:col10/1733584466411/Put/seqid=0 2024-12-07T15:14:26,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742036_1212 (size=14994) 2024-12-07T15:14:26,603 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:26,608 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120719364c1dcb064babb3b73d6fa722f533_b393b7989f53c0069cd9db33cd3a7e39 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120719364c1dcb064babb3b73d6fa722f533_b393b7989f53c0069cd9db33cd3a7e39 2024-12-07T15:14:26,612 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/A/a9345f82a2a94f12b065a7cf87879695, store: [table=TestAcidGuarantees family=A region=b393b7989f53c0069cd9db33cd3a7e39] 2024-12-07T15:14:26,612 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/A/a9345f82a2a94f12b065a7cf87879695 is 175, key is test_row_0/A:col10/1733584466411/Put/seqid=0 2024-12-07T15:14:26,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742037_1213 (size=39949) 2024-12-07T15:14:26,636 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=442, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/A/a9345f82a2a94f12b065a7cf87879695 2024-12-07T15:14:26,651 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/B/fd10cbe195b34bdfbd4a508764d1b3fe is 50, key is test_row_0/B:col10/1733584466411/Put/seqid=0 2024-12-07T15:14:26,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742038_1214 (size=12301) 2024-12-07T15:14:26,683 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:26,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38036 deadline: 1733584526683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:26,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:26,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38006 deadline: 1733584526683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:26,684 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:26,684 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:26,684 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:26,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:26,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38038 deadline: 1733584526683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:26,684 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:26,684 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:26,685 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:26,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:26,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:26,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:26,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:26,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38036 deadline: 1733584526893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:26,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:26,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38006 deadline: 1733584526893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:26,895 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:26,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38038 deadline: 1733584526893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:26,896 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:26,896 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:26,897 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:26,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:26,899 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:26,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:26,899 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:26,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:26,899 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:27,067 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=442 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/B/fd10cbe195b34bdfbd4a508764d1b3fe 2024-12-07T15:14:27,077 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/C/41978fcfae444dd8abf9ec974ec641dc is 50, key is test_row_0/C:col10/1733584466411/Put/seqid=0 2024-12-07T15:14:27,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742039_1215 (size=12301) 2024-12-07T15:14:27,085 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=442 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/C/41978fcfae444dd8abf9ec974ec641dc 2024-12-07T15:14:27,090 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/A/a9345f82a2a94f12b065a7cf87879695 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/a9345f82a2a94f12b065a7cf87879695 2024-12-07T15:14:27,095 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/a9345f82a2a94f12b065a7cf87879695, entries=200, sequenceid=442, filesize=39.0 K 2024-12-07T15:14:27,096 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/B/fd10cbe195b34bdfbd4a508764d1b3fe as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/fd10cbe195b34bdfbd4a508764d1b3fe 2024-12-07T15:14:27,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,108 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/fd10cbe195b34bdfbd4a508764d1b3fe, entries=150, sequenceid=442, filesize=12.0 K 2024-12-07T15:14:27,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,109 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/C/41978fcfae444dd8abf9ec974ec641dc as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/41978fcfae444dd8abf9ec974ec641dc 2024-12-07T15:14:27,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,115 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/41978fcfae444dd8abf9ec974ec641dc, entries=150, sequenceid=442, filesize=12.0 K 2024-12-07T15:14:27,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,116 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~156.54 KB/160300, heapSize ~439.42 KB/449968, currentSize=33.54 KB/34350 for b393b7989f53c0069cd9db33cd3a7e39 in 567ms, sequenceid=442, compaction requested=true 2024-12-07T15:14:27,116 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b393b7989f53c0069cd9db33cd3a7e39: 2024-12-07T15:14:27,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b393b7989f53c0069cd9db33cd3a7e39:A, priority=-2147483648, current under compaction store size is 1 2024-12-07T15:14:27,117 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:14:27,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:14:27,117 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:14:27,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b393b7989f53c0069cd9db33cd3a7e39:B, priority=-2147483648, current under compaction store size is 2 2024-12-07T15:14:27,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:14:27,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b393b7989f53c0069cd9db33cd3a7e39:C, priority=-2147483648, current under compaction store size is 3 2024-12-07T15:14:27,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T15:14:27,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,118 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103175 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:14:27,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,118 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1541): b393b7989f53c0069cd9db33cd3a7e39/A is initiating minor compaction (all files) 2024-12-07T15:14:27,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,118 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b393b7989f53c0069cd9db33cd3a7e39/A in TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:27,118 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/d3e9422e0a64435490077642cba5ba61, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/22b3b6c20eed4a3e9ca1878796f5709b, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/a9345f82a2a94f12b065a7cf87879695] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp, totalSize=100.8 K 2024-12-07T15:14:27,118 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:27,118 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. files: [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/d3e9422e0a64435490077642cba5ba61, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/22b3b6c20eed4a3e9ca1878796f5709b, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/a9345f82a2a94f12b065a7cf87879695] 2024-12-07T15:14:27,119 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:14:27,119 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1541): b393b7989f53c0069cd9db33cd3a7e39/B is initiating minor compaction (all files) 2024-12-07T15:14:27,119 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b393b7989f53c0069cd9db33cd3a7e39/B in TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:27,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,119 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/9a45344881274d838e2d0a8dd8613e9b, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/e770910ee8534813ae759076573b7e58, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/fd10cbe195b34bdfbd4a508764d1b3fe] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp, totalSize=36.7 K 2024-12-07T15:14:27,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,119 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3e9422e0a64435490077642cba5ba61, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1733584463772 2024-12-07T15:14:27,120 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a45344881274d838e2d0a8dd8613e9b, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1733584463772 2024-12-07T15:14:27,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,120 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 22b3b6c20eed4a3e9ca1878796f5709b, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1733584464948 2024-12-07T15:14:27,120 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting e770910ee8534813ae759076573b7e58, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1733584464948 2024-12-07T15:14:27,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,120 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting a9345f82a2a94f12b065a7cf87879695, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=442, earliestPutTs=1733584466411 2024-12-07T15:14:27,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,121 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting fd10cbe195b34bdfbd4a508764d1b3fe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=442, earliestPutTs=1733584466411 2024-12-07T15:14:27,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,130 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b393b7989f53c0069cd9db33cd3a7e39] 2024-12-07T15:14:27,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,132 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241207000b8e5c20b644d384fd28535db4dffc_b393b7989f53c0069cd9db33cd3a7e39 store=[table=TestAcidGuarantees family=A region=b393b7989f53c0069cd9db33cd3a7e39] 2024-12-07T15:14:27,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,132 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b393b7989f53c0069cd9db33cd3a7e39#B#compaction#176 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:14:27,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,133 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/B/6b694f959b6b410bb9cf27c999658073 is 50, key is test_row_0/B:col10/1733584466411/Put/seqid=0 2024-12-07T15:14:27,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,135 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241207000b8e5c20b644d384fd28535db4dffc_b393b7989f53c0069cd9db33cd3a7e39, store=[table=TestAcidGuarantees family=A region=b393b7989f53c0069cd9db33cd3a7e39] 2024-12-07T15:14:27,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,135 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207000b8e5c20b644d384fd28535db4dffc_b393b7989f53c0069cd9db33cd3a7e39 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b393b7989f53c0069cd9db33cd3a7e39] 2024-12-07T15:14:27,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742040_1216 (size=13119) 2024-12-07T15:14:27,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742041_1217 (size=4469) 2024-12-07T15:14:27,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,178 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/B/6b694f959b6b410bb9cf27c999658073 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/6b694f959b6b410bb9cf27c999658073 2024-12-07T15:14:27,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,185 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b393b7989f53c0069cd9db33cd3a7e39/B of b393b7989f53c0069cd9db33cd3a7e39 into 6b694f959b6b410bb9cf27c999658073(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:14:27,185 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b393b7989f53c0069cd9db33cd3a7e39: 2024-12-07T15:14:27,185 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., storeName=b393b7989f53c0069cd9db33cd3a7e39/B, priority=13, startTime=1733584467117; duration=0sec 2024-12-07T15:14:27,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,185 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T15:14:27,185 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b393b7989f53c0069cd9db33cd3a7e39:B 2024-12-07T15:14:27,185 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:14:27,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,186 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:14:27,187 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1541): b393b7989f53c0069cd9db33cd3a7e39/C is initiating minor compaction (all files) 2024-12-07T15:14:27,187 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b393b7989f53c0069cd9db33cd3a7e39/C in TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:27,187 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/7cab36cf861b4b2a9517682255982d42, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/7d83597e3dfa455fb1d954406fc035dc, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/41978fcfae444dd8abf9ec974ec641dc] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp, totalSize=36.7 K 2024-12-07T15:14:27,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,187 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 7cab36cf861b4b2a9517682255982d42, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1733584463772 2024-12-07T15:14:27,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,188 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d83597e3dfa455fb1d954406fc035dc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1733584464948 2024-12-07T15:14:27,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,188 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 41978fcfae444dd8abf9ec974ec641dc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=442, earliestPutTs=1733584466411 2024-12-07T15:14:27,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,199 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b393b7989f53c0069cd9db33cd3a7e39#C#compaction#178 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:14:27,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,199 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/C/4d8f3a611ec9428991da248512971adb is 50, key is test_row_0/C:col10/1733584466411/Put/seqid=0 2024-12-07T15:14:27,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on b393b7989f53c0069cd9db33cd3a7e39 2024-12-07T15:14:27,217 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b393b7989f53c0069cd9db33cd3a7e39 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-07T15:14:27,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b393b7989f53c0069cd9db33cd3a7e39, store=A 2024-12-07T15:14:27,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:14:27,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b393b7989f53c0069cd9db33cd3a7e39, store=B 2024-12-07T15:14:27,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:14:27,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b393b7989f53c0069cd9db33cd3a7e39, store=C 2024-12-07T15:14:27,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:14:27,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742042_1218 (size=13119) 2024-12-07T15:14:27,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,253 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207d2a151fb32ea4aec93ebfe647a0e3b75_b393b7989f53c0069cd9db33cd3a7e39 is 50, key is test_row_0/A:col10/1733584467215/Put/seqid=0 2024-12-07T15:14:27,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742043_1219 (size=14994) 2024-12-07T15:14:27,287 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733584467260, max timestamp=1733584467283Num uniques 859; 2024-12-07T15:14:27,287 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-07T15:14:27,287 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733584467260, max timestamp=1733584467283Num uniques 859; 2024-12-07T15:14:27,287 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-07T15:14:27,289 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733584467260, max timestamp=1733584467283Num uniques 859; 2024-12-07T15:14:27,289 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-07T15:14:27,327 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:27,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38038 deadline: 1733584527317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:27,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:27,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:27,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:27,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:27,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38036 deadline: 1733584527326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:27,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:27,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:27,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:27,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:27,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38006 deadline: 1733584527327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:27,333 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:27,333 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:27,334 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:27,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:27,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38038 deadline: 1733584527434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:27,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:27,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38036 deadline: 1733584527434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:27,442 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:27,442 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:27,442 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:27,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:27,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:27,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:27,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-07T15:14:27,444 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T15:14:27,445 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T15:14:27,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-12-07T15:14:27,447 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T15:14:27,448 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:27,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38006 deadline: 1733584527444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:27,449 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:27,449 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:27,449 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:27,451 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T15:14:27,451 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T15:14:27,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-07T15:14:27,488 INFO [Thread-617 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-617. 2024-12-07T15:14:27,488 DEBUG [Thread-617 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T15:14:27,488 DEBUG [Thread-617 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:14:27,488 DEBUG [Thread-617 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:14:27,488 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T15:14:27,493 INFO [Thread-619 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-619. 2024-12-07T15:14:27,493 DEBUG [Thread-619 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T15:14:27,493 DEBUG [Thread-619 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:14:27,493 DEBUG [Thread-619 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:14:27,493 INFO [Thread-623 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-623. 2024-12-07T15:14:27,493 DEBUG [Thread-623 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T15:14:27,493 DEBUG [Thread-623 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:14:27,494 DEBUG [Thread-623 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:14:27,494 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T15:14:27,498 INFO [Thread-621 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-621. 2024-12-07T15:14:27,498 DEBUG [Thread-621 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T15:14:27,498 DEBUG [Thread-621 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:14:27,498 DEBUG [Thread-621 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:14:27,498 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T15:14:27,500 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T15:14:27,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-07T15:14:27,575 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b393b7989f53c0069cd9db33cd3a7e39#A#compaction#177 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:14:27,576 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/A/d15aa895c7f24d26adca234924e435e4 is 175, key is test_row_0/A:col10/1733584466411/Put/seqid=0 2024-12-07T15:14:27,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742044_1220 (size=32073) 2024-12-07T15:14:27,604 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-07T15:14:27,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:27,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. as already flushing 2024-12-07T15:14:27,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:27,605 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:27,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:27,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:27,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:27,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:27,654 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:27,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38036 deadline: 1733584527653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:27,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38006 deadline: 1733584527654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:27,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38038 deadline: 1733584527653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:27,655 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:27,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:27,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:27,655 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:27,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:27,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:27,655 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:27,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:27,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:27,656 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/C/4d8f3a611ec9428991da248512971adb as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/4d8f3a611ec9428991da248512971adb 2024-12-07T15:14:27,662 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b393b7989f53c0069cd9db33cd3a7e39/C of b393b7989f53c0069cd9db33cd3a7e39 into 4d8f3a611ec9428991da248512971adb(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:14:27,662 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b393b7989f53c0069cd9db33cd3a7e39: 2024-12-07T15:14:27,662 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., storeName=b393b7989f53c0069cd9db33cd3a7e39/C, priority=13, startTime=1733584467117; duration=0sec 2024-12-07T15:14:27,662 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:14:27,662 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b393b7989f53c0069cd9db33cd3a7e39:C 2024-12-07T15:14:27,671 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:27,676 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207d2a151fb32ea4aec93ebfe647a0e3b75_b393b7989f53c0069cd9db33cd3a7e39 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207d2a151fb32ea4aec93ebfe647a0e3b75_b393b7989f53c0069cd9db33cd3a7e39 2024-12-07T15:14:27,677 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/A/0423d466a0444575bbc48eb80e321166, store: [table=TestAcidGuarantees family=A region=b393b7989f53c0069cd9db33cd3a7e39] 2024-12-07T15:14:27,677 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/A/0423d466a0444575bbc48eb80e321166 is 175, key is test_row_0/A:col10/1733584467215/Put/seqid=0 2024-12-07T15:14:27,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742045_1221 (size=39949) 2024-12-07T15:14:27,686 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=453, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/A/0423d466a0444575bbc48eb80e321166 2024-12-07T15:14:27,694 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/B/2938b84e7c7849ee92eb88b926064eda is 50, key is test_row_0/B:col10/1733584467215/Put/seqid=0 2024-12-07T15:14:27,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742046_1222 (size=12301) 2024-12-07T15:14:27,758 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-07T15:14:27,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:27,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. as already flushing 2024-12-07T15:14:27,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:27,759 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:27,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:27,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:27,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-07T15:14:27,912 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-07T15:14:27,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:27,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. as already flushing 2024-12-07T15:14:27,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:27,913 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:27,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:27,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:27,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:27,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:27,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:27,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38006 deadline: 1733584527964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:27,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38036 deadline: 1733584527964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:27,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38038 deadline: 1733584527964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:27,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:27,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:27,965 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:27,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:27,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:27,965 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:27,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:27,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:27,965 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:27,986 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/A/d15aa895c7f24d26adca234924e435e4 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/d15aa895c7f24d26adca234924e435e4 2024-12-07T15:14:27,992 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b393b7989f53c0069cd9db33cd3a7e39/A of b393b7989f53c0069cd9db33cd3a7e39 into d15aa895c7f24d26adca234924e435e4(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:14:27,992 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b393b7989f53c0069cd9db33cd3a7e39: 2024-12-07T15:14:27,992 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., storeName=b393b7989f53c0069cd9db33cd3a7e39/A, priority=13, startTime=1733584467116; duration=0sec 2024-12-07T15:14:27,992 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:14:27,992 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b393b7989f53c0069cd9db33cd3a7e39:A 2024-12-07T15:14:28,066 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-07T15:14:28,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:28,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. as already flushing 2024-12-07T15:14:28,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:28,067 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:28,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:28,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:28,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-07T15:14:28,109 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=453 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/B/2938b84e7c7849ee92eb88b926064eda 2024-12-07T15:14:28,116 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/C/bd0461a59d134fe8aa6bfb7f8712ad7d is 50, key is test_row_0/C:col10/1733584467215/Put/seqid=0 2024-12-07T15:14:28,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742047_1223 (size=12301) 2024-12-07T15:14:28,220 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-07T15:14:28,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:28,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. as already flushing 2024-12-07T15:14:28,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:28,220 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:28,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:28,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:28,374 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-07T15:14:28,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:28,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. as already flushing 2024-12-07T15:14:28,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:28,374 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:28,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:28,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:28,474 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:28,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:28,474 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:28,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38038 deadline: 1733584528473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:28,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38006 deadline: 1733584528473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:28,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38036 deadline: 1733584528474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:28,474 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:28,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:28,474 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:28,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:28,474 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:28,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:28,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:28,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:28,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:14:28,521 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=453 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/C/bd0461a59d134fe8aa6bfb7f8712ad7d 2024-12-07T15:14:28,527 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-07T15:14:28,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:28,527 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/A/0423d466a0444575bbc48eb80e321166 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/0423d466a0444575bbc48eb80e321166 2024-12-07T15:14:28,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. as already flushing 2024-12-07T15:14:28,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:28,527 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:28,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:28,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:28,534 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/0423d466a0444575bbc48eb80e321166, entries=200, sequenceid=453, filesize=39.0 K 2024-12-07T15:14:28,535 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/B/2938b84e7c7849ee92eb88b926064eda as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/2938b84e7c7849ee92eb88b926064eda 2024-12-07T15:14:28,543 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/2938b84e7c7849ee92eb88b926064eda, entries=150, sequenceid=453, filesize=12.0 K 2024-12-07T15:14:28,544 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/C/bd0461a59d134fe8aa6bfb7f8712ad7d as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/bd0461a59d134fe8aa6bfb7f8712ad7d 2024-12-07T15:14:28,549 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/bd0461a59d134fe8aa6bfb7f8712ad7d, entries=150, sequenceid=453, filesize=12.0 K 2024-12-07T15:14:28,551 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=131.94 KB/135110 for b393b7989f53c0069cd9db33cd3a7e39 in 1333ms, sequenceid=453, compaction requested=false 2024-12-07T15:14:28,551 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b393b7989f53c0069cd9db33cd3a7e39: 2024-12-07T15:14:28,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-07T15:14:28,681 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-07T15:14:28,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:28,681 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing b393b7989f53c0069cd9db33cd3a7e39 3/3 column families, dataSize=131.94 KB heapSize=375.22 KB 2024-12-07T15:14:28,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b393b7989f53c0069cd9db33cd3a7e39, store=A 2024-12-07T15:14:28,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T15:14:28,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b393b7989f53c0069cd9db33cd3a7e39, store=B 2024-12-07T15:14:28,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T15:14:28,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b393b7989f53c0069cd9db33cd3a7e39, store=C 2024-12-07T15:14:28,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T15:14:28,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207072ce1d4ea7c46c197c473596448b247_b393b7989f53c0069cd9db33cd3a7e39 is 50, key is test_row_0/A:col10/1733584467314/Put/seqid=0 2024-12-07T15:14:28,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742048_1224 (size=12454) 2024-12-07T15:14:29,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:29,096 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207072ce1d4ea7c46c197c473596448b247_b393b7989f53c0069cd9db33cd3a7e39 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207072ce1d4ea7c46c197c473596448b247_b393b7989f53c0069cd9db33cd3a7e39 2024-12-07T15:14:29,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/A/7cda4208cbc44caead0b6bb1d5fa8095, store: [table=TestAcidGuarantees family=A region=b393b7989f53c0069cd9db33cd3a7e39] 2024-12-07T15:14:29,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/A/7cda4208cbc44caead0b6bb1d5fa8095 is 175, key is test_row_0/A:col10/1733584467314/Put/seqid=0 2024-12-07T15:14:29,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742049_1225 (size=31255) 2024-12-07T15:14:29,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on b393b7989f53c0069cd9db33cd3a7e39 2024-12-07T15:14:29,485 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. as already flushing 2024-12-07T15:14:29,485 INFO [Thread-608 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-608. 2024-12-07T15:14:29,485 DEBUG [Thread-608 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T15:14:29,485 DEBUG [Thread-608 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:14:29,485 INFO [Thread-610 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-610. 2024-12-07T15:14:29,485 DEBUG [Thread-610 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T15:14:29,485 DEBUG [Thread-608 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:14:29,485 DEBUG [Thread-610 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:14:29,485 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T15:14:29,485 DEBUG [Thread-610 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:14:29,486 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T15:14:29,494 INFO [Thread-612 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-612. 2024-12-07T15:14:29,494 DEBUG [Thread-612 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T15:14:29,494 DEBUG [Thread-612 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:14:29,494 DEBUG [Thread-612 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:14:29,494 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T15:14:29,502 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=491, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/A/7cda4208cbc44caead0b6bb1d5fa8095 2024-12-07T15:14:29,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/B/5db8619ed2ab4d0fbe7b2edd460a2116 is 50, key is test_row_0/B:col10/1733584467314/Put/seqid=0 2024-12-07T15:14:29,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742050_1226 (size=12301) 2024-12-07T15:14:29,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-07T15:14:29,916 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=491 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/B/5db8619ed2ab4d0fbe7b2edd460a2116 2024-12-07T15:14:29,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/C/28237587e9434699a930ce147536aea4 is 50, key is test_row_0/C:col10/1733584467314/Put/seqid=0 2024-12-07T15:14:29,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742051_1227 (size=12301) 2024-12-07T15:14:30,330 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=491 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/C/28237587e9434699a930ce147536aea4 2024-12-07T15:14:30,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/A/7cda4208cbc44caead0b6bb1d5fa8095 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/7cda4208cbc44caead0b6bb1d5fa8095 2024-12-07T15:14:30,338 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/7cda4208cbc44caead0b6bb1d5fa8095, entries=150, sequenceid=491, filesize=30.5 K 2024-12-07T15:14:30,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/B/5db8619ed2ab4d0fbe7b2edd460a2116 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/5db8619ed2ab4d0fbe7b2edd460a2116 2024-12-07T15:14:30,343 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/5db8619ed2ab4d0fbe7b2edd460a2116, entries=150, sequenceid=491, filesize=12.0 K 2024-12-07T15:14:30,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/C/28237587e9434699a930ce147536aea4 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/28237587e9434699a930ce147536aea4 2024-12-07T15:14:30,348 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/28237587e9434699a930ce147536aea4, entries=150, sequenceid=491, filesize=12.0 K 2024-12-07T15:14:30,349 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~131.94 KB/135110, heapSize ~375.17 KB/384176, currentSize=20.13 KB/20610 for b393b7989f53c0069cd9db33cd3a7e39 in 1668ms, sequenceid=491, compaction requested=true 2024-12-07T15:14:30,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for b393b7989f53c0069cd9db33cd3a7e39: 2024-12-07T15:14:30,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:30,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-12-07T15:14:30,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-12-07T15:14:30,352 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-12-07T15:14:30,352 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9000 sec 2024-12-07T15:14:30,354 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 2.9080 sec 2024-12-07T15:14:31,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-07T15:14:31,604 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T15:14:34,325 INFO [Thread-606 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-606. 2024-12-07T15:14:34,325 DEBUG [Thread-606 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T15:14:34,325 DEBUG [Thread-606 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:14:34,325 DEBUG [Thread-606 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:14:34,325 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T15:14:34,375 INFO [Thread-614 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-614. 2024-12-07T15:14:34,376 DEBUG [Thread-614 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T15:14:34,376 DEBUG [Thread-614 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:14:34,376 DEBUG [Thread-614 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:14:34,376 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T15:14:34,376 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-07T15:14:34,376 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 35 2024-12-07T15:14:34,376 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 118 2024-12-07T15:14:34,376 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 108 2024-12-07T15:14:34,376 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 109 2024-12-07T15:14:34,376 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 41 2024-12-07T15:14:34,376 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-07T15:14:34,376 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2690 2024-12-07T15:14:34,376 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2522 2024-12-07T15:14:34,376 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-07T15:14:34,376 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1150 2024-12-07T15:14:34,376 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3445 rows 2024-12-07T15:14:34,376 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1148 2024-12-07T15:14:34,376 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3442 rows 2024-12-07T15:14:34,377 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-07T15:14:34,377 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-07T15:14:34,377 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:14:34,377 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:14:34,377 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T15:14:34,378 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-07T15:14:34,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-07T15:14:34,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-07T15:14:34,382 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733584474382"}]},"ts":"1733584474382"} 2024-12-07T15:14:34,384 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-07T15:14:34,384 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-07T15:14:34,385 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-07T15:14:34,387 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b393b7989f53c0069cd9db33cd3a7e39, UNASSIGN}] 2024-12-07T15:14:34,389 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b393b7989f53c0069cd9db33cd3a7e39, UNASSIGN 2024-12-07T15:14:34,390 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=b393b7989f53c0069cd9db33cd3a7e39, regionState=CLOSING, regionLocation=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:34,392 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b393b7989f53c0069cd9db33cd3a7e39, UNASSIGN because future has completed 2024-12-07T15:14:34,393 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-07T15:14:34,393 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; CloseRegionProcedure b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401}] 2024-12-07T15:14:34,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-07T15:14:34,546 INFO [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(122): Close b393b7989f53c0069cd9db33cd3a7e39 2024-12-07T15:14:34,546 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-07T15:14:34,546 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1722): Closing b393b7989f53c0069cd9db33cd3a7e39, disabling compactions & flushes 2024-12-07T15:14:34,546 INFO [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:34,546 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:34,546 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. after waiting 0 ms 2024-12-07T15:14:34,546 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:34,546 INFO [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(2902): Flushing b393b7989f53c0069cd9db33cd3a7e39 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-07T15:14:34,547 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b393b7989f53c0069cd9db33cd3a7e39, store=A 2024-12-07T15:14:34,547 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:14:34,547 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b393b7989f53c0069cd9db33cd3a7e39, store=B 2024-12-07T15:14:34,547 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:14:34,547 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b393b7989f53c0069cd9db33cd3a7e39, store=C 2024-12-07T15:14:34,547 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:14:34,553 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120736d958b54ec7499b8904ee529e00e937_b393b7989f53c0069cd9db33cd3a7e39 is 50, key is test_row_1/A:col10/1733584474324/Put/seqid=0 2024-12-07T15:14:34,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742052_1228 (size=9914) 2024-12-07T15:14:34,560 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:34,566 INFO [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120736d958b54ec7499b8904ee529e00e937_b393b7989f53c0069cd9db33cd3a7e39 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120736d958b54ec7499b8904ee529e00e937_b393b7989f53c0069cd9db33cd3a7e39 2024-12-07T15:14:34,567 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/A/a93655faf1eb4887966672d14cde7926, store: [table=TestAcidGuarantees family=A region=b393b7989f53c0069cd9db33cd3a7e39] 2024-12-07T15:14:34,567 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/A/a93655faf1eb4887966672d14cde7926 is 175, key is test_row_1/A:col10/1733584474324/Put/seqid=0 2024-12-07T15:14:34,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742053_1229 (size=22561) 2024-12-07T15:14:34,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-07T15:14:34,972 INFO [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=499, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/A/a93655faf1eb4887966672d14cde7926 2024-12-07T15:14:34,978 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/B/114c4515b26e473c965de1351a391734 is 50, key is test_row_1/B:col10/1733584474324/Put/seqid=0 2024-12-07T15:14:34,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742054_1230 (size=9857) 2024-12-07T15:14:35,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-07T15:14:35,383 INFO [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=499 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/B/114c4515b26e473c965de1351a391734 2024-12-07T15:14:35,391 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/C/0fc1733b31384288bcf851eca124c9c3 is 50, key is test_row_1/C:col10/1733584474324/Put/seqid=0 2024-12-07T15:14:35,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742055_1231 (size=9857) 2024-12-07T15:14:35,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-07T15:14:35,799 INFO [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=499 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/C/0fc1733b31384288bcf851eca124c9c3 2024-12-07T15:14:35,803 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/A/a93655faf1eb4887966672d14cde7926 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/a93655faf1eb4887966672d14cde7926 2024-12-07T15:14:35,806 INFO [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/a93655faf1eb4887966672d14cde7926, entries=100, sequenceid=499, filesize=22.0 K 2024-12-07T15:14:35,807 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/B/114c4515b26e473c965de1351a391734 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/114c4515b26e473c965de1351a391734 2024-12-07T15:14:35,810 INFO [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/114c4515b26e473c965de1351a391734, entries=100, sequenceid=499, filesize=9.6 K 2024-12-07T15:14:35,811 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/.tmp/C/0fc1733b31384288bcf851eca124c9c3 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/0fc1733b31384288bcf851eca124c9c3 2024-12-07T15:14:35,815 INFO [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/0fc1733b31384288bcf851eca124c9c3, entries=100, sequenceid=499, filesize=9.6 K 2024-12-07T15:14:35,815 INFO [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for b393b7989f53c0069cd9db33cd3a7e39 in 1269ms, sequenceid=499, compaction requested=true 2024-12-07T15:14:35,816 DEBUG [StoreCloser-TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/1971927d23774c0b8118642758dd550d, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/9e9a6a252298461990ada271d3d40639, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/808e0e4e7dbe4b558f1ea1218a87e52b, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/b38b6d7881ed4f6591553ed7d0524177, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/86dac675f9954267b100910c6d57cda6, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/c1a15f2e0802400c99c942950da40098, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/963f1c81ec604ccdbf35aa50cf584592, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/c5a156582a3446ad88d8c14897191d62, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/a84ed360131f424999ab322f82d81d7e, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/2d90f70a3fc04c049ca5385d8e4ced6c, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/ef20962a0815447a937e2beaec252bb7, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/cfff55a4dae24ae698d87826c790b908, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/247419885c4646a5b651a79b07005e99, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/b76cfbd63a1c4c3cac9af7473155a27d, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/1c61771a38004224bd44cf2faa552483, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/73d8a1eff7a34169ad538f47feb9405e, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/debab29f5ec241cca4a8e88c768e37b6, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/e1e4c4770e454a699b1778fb17db18ff, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/7a4c75f2a9aa49c1a32f59f246603249, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/68b5c8fce8e84b1f932367853b81adf2, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/9668f3d9ebbd4224b094cf8282421e6a, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/d3e9422e0a64435490077642cba5ba61, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/22b3b6c20eed4a3e9ca1878796f5709b, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/a9345f82a2a94f12b065a7cf87879695] to archive 2024-12-07T15:14:35,817 DEBUG [StoreCloser-TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-07T15:14:35,818 DEBUG [StoreCloser-TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/1971927d23774c0b8118642758dd550d to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/1971927d23774c0b8118642758dd550d 2024-12-07T15:14:35,819 DEBUG [StoreCloser-TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/9e9a6a252298461990ada271d3d40639 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/9e9a6a252298461990ada271d3d40639 2024-12-07T15:14:35,820 DEBUG [StoreCloser-TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/808e0e4e7dbe4b558f1ea1218a87e52b to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/808e0e4e7dbe4b558f1ea1218a87e52b 2024-12-07T15:14:35,822 DEBUG [StoreCloser-TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/b38b6d7881ed4f6591553ed7d0524177 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/b38b6d7881ed4f6591553ed7d0524177 2024-12-07T15:14:35,823 DEBUG [StoreCloser-TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/86dac675f9954267b100910c6d57cda6 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/86dac675f9954267b100910c6d57cda6 2024-12-07T15:14:35,825 DEBUG [StoreCloser-TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/c1a15f2e0802400c99c942950da40098 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/c1a15f2e0802400c99c942950da40098 2024-12-07T15:14:35,826 DEBUG [StoreCloser-TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/963f1c81ec604ccdbf35aa50cf584592 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/963f1c81ec604ccdbf35aa50cf584592 2024-12-07T15:14:35,828 DEBUG [StoreCloser-TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/c5a156582a3446ad88d8c14897191d62 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/c5a156582a3446ad88d8c14897191d62 2024-12-07T15:14:35,829 DEBUG [StoreCloser-TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/a84ed360131f424999ab322f82d81d7e to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/a84ed360131f424999ab322f82d81d7e 2024-12-07T15:14:35,830 DEBUG [StoreCloser-TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/2d90f70a3fc04c049ca5385d8e4ced6c to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/2d90f70a3fc04c049ca5385d8e4ced6c 2024-12-07T15:14:35,831 DEBUG [StoreCloser-TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/ef20962a0815447a937e2beaec252bb7 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/ef20962a0815447a937e2beaec252bb7 2024-12-07T15:14:35,832 DEBUG [StoreCloser-TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/cfff55a4dae24ae698d87826c790b908 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/cfff55a4dae24ae698d87826c790b908 2024-12-07T15:14:35,834 DEBUG [StoreCloser-TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/247419885c4646a5b651a79b07005e99 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/247419885c4646a5b651a79b07005e99 2024-12-07T15:14:35,835 DEBUG [StoreCloser-TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/b76cfbd63a1c4c3cac9af7473155a27d to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/b76cfbd63a1c4c3cac9af7473155a27d 2024-12-07T15:14:35,836 DEBUG [StoreCloser-TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/1c61771a38004224bd44cf2faa552483 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/1c61771a38004224bd44cf2faa552483 2024-12-07T15:14:35,838 DEBUG [StoreCloser-TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/73d8a1eff7a34169ad538f47feb9405e to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/73d8a1eff7a34169ad538f47feb9405e 2024-12-07T15:14:35,839 DEBUG [StoreCloser-TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/debab29f5ec241cca4a8e88c768e37b6 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/debab29f5ec241cca4a8e88c768e37b6 2024-12-07T15:14:35,840 DEBUG [StoreCloser-TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/e1e4c4770e454a699b1778fb17db18ff to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/e1e4c4770e454a699b1778fb17db18ff 2024-12-07T15:14:35,841 DEBUG [StoreCloser-TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/7a4c75f2a9aa49c1a32f59f246603249 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/7a4c75f2a9aa49c1a32f59f246603249 2024-12-07T15:14:35,842 DEBUG [StoreCloser-TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/68b5c8fce8e84b1f932367853b81adf2 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/68b5c8fce8e84b1f932367853b81adf2 2024-12-07T15:14:35,843 DEBUG [StoreCloser-TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/9668f3d9ebbd4224b094cf8282421e6a to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/9668f3d9ebbd4224b094cf8282421e6a 2024-12-07T15:14:35,844 DEBUG [StoreCloser-TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/d3e9422e0a64435490077642cba5ba61 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/d3e9422e0a64435490077642cba5ba61 2024-12-07T15:14:35,845 DEBUG [StoreCloser-TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/22b3b6c20eed4a3e9ca1878796f5709b to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/22b3b6c20eed4a3e9ca1878796f5709b 2024-12-07T15:14:35,846 DEBUG [StoreCloser-TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/a9345f82a2a94f12b065a7cf87879695 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/a9345f82a2a94f12b065a7cf87879695 2024-12-07T15:14:35,848 DEBUG [StoreCloser-TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/8b1f614b9b444a22817914dae302cd4e, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/4fb8bff65a0941a6a882157b5253326c, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/929db56ce8cc4600933bc5209587d6f6, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/26861b39ff434574954ac1130e531dff, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/7d23a3261a3d4d149c89d6b1a002209a, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/552958a789f94d69830a836c4e84f4bd, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/b8d5848f24444b32b6c89250a5c75274, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/92eaac8ad8174f4a9fec575d3136cbc1, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/b030904bef4745bf93ddae147deb18da, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/1f26df873f9e4232880a41c38086017c, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/4ab5ecc506c24e14b8c05c9cfbd4cf7c, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/1f2306b2566d486aa7b3171708f6f098, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/88594334250a43ec8fca86bb43720f2a, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/e000b6d727814135aa3c6b38b2758961, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/ec627eac599e4eaaab83d12575fc00ce, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/6e18442b5c234b2397e183e20d5aff63, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/e3952bf2c768468a8364dcfab134a557, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/7140895f2ce441f98d14e666b2e58e47, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/38b157532f8a44de9dc61881a2f2fd80, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/be6f890b87fb41cc9f0b5db4b56cb9be, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/9a45344881274d838e2d0a8dd8613e9b, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/0786bd92ae8e4c1085ccca93df4c68d4, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/e770910ee8534813ae759076573b7e58, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/fd10cbe195b34bdfbd4a508764d1b3fe] to archive 2024-12-07T15:14:35,848 DEBUG [StoreCloser-TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-07T15:14:35,850 DEBUG [StoreCloser-TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/8b1f614b9b444a22817914dae302cd4e to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/8b1f614b9b444a22817914dae302cd4e 2024-12-07T15:14:35,851 DEBUG [StoreCloser-TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/4fb8bff65a0941a6a882157b5253326c to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/4fb8bff65a0941a6a882157b5253326c 2024-12-07T15:14:35,852 DEBUG [StoreCloser-TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/929db56ce8cc4600933bc5209587d6f6 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/929db56ce8cc4600933bc5209587d6f6 2024-12-07T15:14:35,852 DEBUG [StoreCloser-TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/26861b39ff434574954ac1130e531dff to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/26861b39ff434574954ac1130e531dff 2024-12-07T15:14:35,853 DEBUG [StoreCloser-TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/7d23a3261a3d4d149c89d6b1a002209a to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/7d23a3261a3d4d149c89d6b1a002209a 2024-12-07T15:14:35,854 DEBUG [StoreCloser-TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/552958a789f94d69830a836c4e84f4bd to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/552958a789f94d69830a836c4e84f4bd 2024-12-07T15:14:35,855 DEBUG [StoreCloser-TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/b8d5848f24444b32b6c89250a5c75274 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/b8d5848f24444b32b6c89250a5c75274 2024-12-07T15:14:35,856 DEBUG [StoreCloser-TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/92eaac8ad8174f4a9fec575d3136cbc1 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/92eaac8ad8174f4a9fec575d3136cbc1 2024-12-07T15:14:35,857 DEBUG [StoreCloser-TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/b030904bef4745bf93ddae147deb18da to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/b030904bef4745bf93ddae147deb18da 2024-12-07T15:14:35,858 DEBUG [StoreCloser-TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/1f26df873f9e4232880a41c38086017c to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/1f26df873f9e4232880a41c38086017c 2024-12-07T15:14:35,859 DEBUG [StoreCloser-TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/4ab5ecc506c24e14b8c05c9cfbd4cf7c to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/4ab5ecc506c24e14b8c05c9cfbd4cf7c 2024-12-07T15:14:35,860 DEBUG [StoreCloser-TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/1f2306b2566d486aa7b3171708f6f098 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/1f2306b2566d486aa7b3171708f6f098 2024-12-07T15:14:35,861 DEBUG [StoreCloser-TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/88594334250a43ec8fca86bb43720f2a to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/88594334250a43ec8fca86bb43720f2a 2024-12-07T15:14:35,862 DEBUG [StoreCloser-TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/e000b6d727814135aa3c6b38b2758961 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/e000b6d727814135aa3c6b38b2758961 2024-12-07T15:14:35,863 DEBUG [StoreCloser-TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/ec627eac599e4eaaab83d12575fc00ce to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/ec627eac599e4eaaab83d12575fc00ce 2024-12-07T15:14:35,864 DEBUG [StoreCloser-TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/6e18442b5c234b2397e183e20d5aff63 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/6e18442b5c234b2397e183e20d5aff63 2024-12-07T15:14:35,865 DEBUG [StoreCloser-TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/e3952bf2c768468a8364dcfab134a557 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/e3952bf2c768468a8364dcfab134a557 2024-12-07T15:14:35,865 DEBUG [StoreCloser-TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/7140895f2ce441f98d14e666b2e58e47 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/7140895f2ce441f98d14e666b2e58e47 2024-12-07T15:14:35,866 DEBUG [StoreCloser-TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/38b157532f8a44de9dc61881a2f2fd80 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/38b157532f8a44de9dc61881a2f2fd80 2024-12-07T15:14:35,867 DEBUG [StoreCloser-TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/be6f890b87fb41cc9f0b5db4b56cb9be to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/be6f890b87fb41cc9f0b5db4b56cb9be 2024-12-07T15:14:35,868 DEBUG [StoreCloser-TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/9a45344881274d838e2d0a8dd8613e9b to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/9a45344881274d838e2d0a8dd8613e9b 2024-12-07T15:14:35,869 DEBUG [StoreCloser-TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/0786bd92ae8e4c1085ccca93df4c68d4 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/0786bd92ae8e4c1085ccca93df4c68d4 2024-12-07T15:14:35,870 DEBUG [StoreCloser-TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/e770910ee8534813ae759076573b7e58 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/e770910ee8534813ae759076573b7e58 2024-12-07T15:14:35,871 DEBUG [StoreCloser-TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/fd10cbe195b34bdfbd4a508764d1b3fe to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/fd10cbe195b34bdfbd4a508764d1b3fe 2024-12-07T15:14:35,873 DEBUG [StoreCloser-TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/2d9caaf98594437fbddfb9b2815d6a5c, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/a78ba985d893400085ff7ff7316f8962, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/e5ce57d984614cf993f7855d8222857a, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/c86814276f8448b0b60387abe9dd8615, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/710867729db542fa81939e889c8c6f96, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/a247c9568ce542f2a817d6e46be55dc1, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/ea8b8007c11e4c6c991d1bfc22df3f3e, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/9141de3767344ab09d9b1c671525f76a, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/86b7d3d216f94081a38c6bf54d203b90, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/6868d4bd16bd4a6ba9f4b0314fc71a97, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/a1bada73e4894b859954823068ebd7f1, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/920149c45f5d442b900068d7311c8a22, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/fb76d6ae7a1e4924b2ee5721322c60a9, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/96f8121fa74b460eb6249314f02d8c6b, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/dacfa487b0ca4b08a1c5a53762bc4581, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/5650c4e397e245c0aff67bca28e594e9, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/733edc4ef2054a7b99597fc8e11c9a12, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/79b408ba5bca402b89bc567337bc014f, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/b515dcbfe9024e0d9a3c73f1b5d8d8f8, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/a575fb48387f425b95ef505b2a794cb7, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/7cab36cf861b4b2a9517682255982d42, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/e1ae27d46caa46e0aa05fa2a7308b4e8, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/7d83597e3dfa455fb1d954406fc035dc, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/41978fcfae444dd8abf9ec974ec641dc] to archive 2024-12-07T15:14:35,874 DEBUG [StoreCloser-TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-07T15:14:35,875 DEBUG [StoreCloser-TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/2d9caaf98594437fbddfb9b2815d6a5c to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/2d9caaf98594437fbddfb9b2815d6a5c 2024-12-07T15:14:35,876 DEBUG [StoreCloser-TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/a78ba985d893400085ff7ff7316f8962 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/a78ba985d893400085ff7ff7316f8962 2024-12-07T15:14:35,877 DEBUG [StoreCloser-TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/e5ce57d984614cf993f7855d8222857a to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/e5ce57d984614cf993f7855d8222857a 2024-12-07T15:14:35,878 DEBUG [StoreCloser-TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/c86814276f8448b0b60387abe9dd8615 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/c86814276f8448b0b60387abe9dd8615 2024-12-07T15:14:35,878 DEBUG [StoreCloser-TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/710867729db542fa81939e889c8c6f96 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/710867729db542fa81939e889c8c6f96 2024-12-07T15:14:35,879 DEBUG [StoreCloser-TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/a247c9568ce542f2a817d6e46be55dc1 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/a247c9568ce542f2a817d6e46be55dc1 2024-12-07T15:14:35,880 DEBUG [StoreCloser-TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/ea8b8007c11e4c6c991d1bfc22df3f3e to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/ea8b8007c11e4c6c991d1bfc22df3f3e 2024-12-07T15:14:35,881 DEBUG [StoreCloser-TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/9141de3767344ab09d9b1c671525f76a to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/9141de3767344ab09d9b1c671525f76a 2024-12-07T15:14:35,882 DEBUG [StoreCloser-TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/86b7d3d216f94081a38c6bf54d203b90 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/86b7d3d216f94081a38c6bf54d203b90 2024-12-07T15:14:35,883 DEBUG [StoreCloser-TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/6868d4bd16bd4a6ba9f4b0314fc71a97 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/6868d4bd16bd4a6ba9f4b0314fc71a97 2024-12-07T15:14:35,884 DEBUG [StoreCloser-TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/a1bada73e4894b859954823068ebd7f1 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/a1bada73e4894b859954823068ebd7f1 2024-12-07T15:14:35,885 DEBUG [StoreCloser-TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/920149c45f5d442b900068d7311c8a22 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/920149c45f5d442b900068d7311c8a22 2024-12-07T15:14:35,886 DEBUG [StoreCloser-TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/fb76d6ae7a1e4924b2ee5721322c60a9 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/fb76d6ae7a1e4924b2ee5721322c60a9 2024-12-07T15:14:35,887 DEBUG [StoreCloser-TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/96f8121fa74b460eb6249314f02d8c6b to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/96f8121fa74b460eb6249314f02d8c6b 2024-12-07T15:14:35,888 DEBUG [StoreCloser-TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/dacfa487b0ca4b08a1c5a53762bc4581 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/dacfa487b0ca4b08a1c5a53762bc4581 2024-12-07T15:14:35,889 DEBUG [StoreCloser-TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/5650c4e397e245c0aff67bca28e594e9 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/5650c4e397e245c0aff67bca28e594e9 2024-12-07T15:14:35,890 DEBUG [StoreCloser-TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/733edc4ef2054a7b99597fc8e11c9a12 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/733edc4ef2054a7b99597fc8e11c9a12 2024-12-07T15:14:35,891 DEBUG [StoreCloser-TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/79b408ba5bca402b89bc567337bc014f to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/79b408ba5bca402b89bc567337bc014f 2024-12-07T15:14:35,894 DEBUG [StoreCloser-TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/b515dcbfe9024e0d9a3c73f1b5d8d8f8 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/b515dcbfe9024e0d9a3c73f1b5d8d8f8 2024-12-07T15:14:35,899 DEBUG [StoreCloser-TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/a575fb48387f425b95ef505b2a794cb7 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/a575fb48387f425b95ef505b2a794cb7 2024-12-07T15:14:35,901 DEBUG [StoreCloser-TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/7cab36cf861b4b2a9517682255982d42 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/7cab36cf861b4b2a9517682255982d42 2024-12-07T15:14:35,903 DEBUG [StoreCloser-TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/e1ae27d46caa46e0aa05fa2a7308b4e8 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/e1ae27d46caa46e0aa05fa2a7308b4e8 2024-12-07T15:14:35,904 DEBUG [StoreCloser-TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/7d83597e3dfa455fb1d954406fc035dc to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/7d83597e3dfa455fb1d954406fc035dc 2024-12-07T15:14:35,906 DEBUG [StoreCloser-TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/41978fcfae444dd8abf9ec974ec641dc to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/41978fcfae444dd8abf9ec974ec641dc 2024-12-07T15:14:35,911 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/recovered.edits/502.seqid, newMaxSeqId=502, maxSeqId=4 2024-12-07T15:14:35,912 INFO [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. 2024-12-07T15:14:35,912 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1676): Region close journal for b393b7989f53c0069cd9db33cd3a7e39: Waiting for close lock at 1733584474546Running coprocessor pre-close hooks at 1733584474546Disabling compacts and flushes for region at 1733584474546Disabling writes for close at 1733584474546Obtaining lock to block concurrent updates at 1733584474546Preparing flush snapshotting stores in b393b7989f53c0069cd9db33cd3a7e39 at 1733584474546Finished memstore snapshotting TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1733584474547 (+1 ms)Flushing stores of TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39. at 1733584474547Flushing b393b7989f53c0069cd9db33cd3a7e39/A: creating writer at 1733584474547Flushing mob file b393b7989f53c0069cd9db33cd3a7e39/A: appending metadata at 1733584474553 (+6 ms)Flushing mob file b393b7989f53c0069cd9db33cd3a7e39/A: closing flushed file at 1733584474553Flushing b393b7989f53c0069cd9db33cd3a7e39/A: appending metadata at 1733584474567 (+14 ms)Flushing b393b7989f53c0069cd9db33cd3a7e39/A: closing flushed file at 1733584474567Flushing b393b7989f53c0069cd9db33cd3a7e39/B: creating writer at 1733584474975 (+408 ms)Flushing b393b7989f53c0069cd9db33cd3a7e39/B: appending metadata at 1733584474978 (+3 ms)Flushing b393b7989f53c0069cd9db33cd3a7e39/B: closing flushed file at 1733584474978Flushing b393b7989f53c0069cd9db33cd3a7e39/C: creating writer at 1733584475387 (+409 ms)Flushing b393b7989f53c0069cd9db33cd3a7e39/C: appending metadata at 1733584475390 (+3 ms)Flushing b393b7989f53c0069cd9db33cd3a7e39/C: closing flushed file at 1733584475390Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2d3fed62: reopening flushed file at 1733584475803 (+413 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@21b274e0: reopening flushed file at 1733584475806 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@53206f4: reopening flushed file at 1733584475811 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for b393b7989f53c0069cd9db33cd3a7e39 in 1269ms, sequenceid=499, compaction requested=true at 1733584475815 (+4 ms)Writing region close event to WAL at 1733584475907 (+92 ms)Running coprocessor post-close hooks at 1733584475911 (+4 ms)Closed at 1733584475912 (+1 ms) 2024-12-07T15:14:35,914 INFO [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(157): Closed b393b7989f53c0069cd9db33cd3a7e39 2024-12-07T15:14:35,914 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=b393b7989f53c0069cd9db33cd3a7e39, regionState=CLOSED 2024-12-07T15:14:35,915 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, ppid=56, state=RUNNABLE, hasLock=false; CloseRegionProcedure b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 because future has completed 2024-12-07T15:14:35,918 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-12-07T15:14:35,918 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; CloseRegionProcedure b393b7989f53c0069cd9db33cd3a7e39, server=d6797ebe9c52,46009,1733584418401 in 1.5230 sec 2024-12-07T15:14:35,920 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=55 2024-12-07T15:14:35,920 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=55, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b393b7989f53c0069cd9db33cd3a7e39, UNASSIGN in 1.5310 sec 2024-12-07T15:14:35,922 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-12-07T15:14:35,922 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5360 sec 2024-12-07T15:14:35,924 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733584475924"}]},"ts":"1733584475924"} 2024-12-07T15:14:35,925 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-07T15:14:35,925 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-07T15:14:35,927 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5480 sec 2024-12-07T15:14:36,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-07T15:14:36,524 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-07T15:14:36,525 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-07T15:14:36,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-07T15:14:36,527 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=58, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-07T15:14:36,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-07T15:14:36,528 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=58, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-07T15:14:36,533 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39 2024-12-07T15:14:36,536 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A, FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B, FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C, FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/recovered.edits] 2024-12-07T15:14:36,540 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/0423d466a0444575bbc48eb80e321166 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/0423d466a0444575bbc48eb80e321166 2024-12-07T15:14:36,542 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/7cda4208cbc44caead0b6bb1d5fa8095 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/7cda4208cbc44caead0b6bb1d5fa8095 2024-12-07T15:14:36,544 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/a93655faf1eb4887966672d14cde7926 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/a93655faf1eb4887966672d14cde7926 2024-12-07T15:14:36,546 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/d15aa895c7f24d26adca234924e435e4 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/A/d15aa895c7f24d26adca234924e435e4 2024-12-07T15:14:36,549 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/114c4515b26e473c965de1351a391734 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/114c4515b26e473c965de1351a391734 2024-12-07T15:14:36,551 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/2938b84e7c7849ee92eb88b926064eda to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/2938b84e7c7849ee92eb88b926064eda 2024-12-07T15:14:36,553 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/5db8619ed2ab4d0fbe7b2edd460a2116 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/5db8619ed2ab4d0fbe7b2edd460a2116 2024-12-07T15:14:36,555 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/6b694f959b6b410bb9cf27c999658073 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/B/6b694f959b6b410bb9cf27c999658073 2024-12-07T15:14:36,558 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/0fc1733b31384288bcf851eca124c9c3 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/0fc1733b31384288bcf851eca124c9c3 2024-12-07T15:14:36,559 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/28237587e9434699a930ce147536aea4 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/28237587e9434699a930ce147536aea4 2024-12-07T15:14:36,560 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/4d8f3a611ec9428991da248512971adb to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/4d8f3a611ec9428991da248512971adb 2024-12-07T15:14:36,562 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/bd0461a59d134fe8aa6bfb7f8712ad7d to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/C/bd0461a59d134fe8aa6bfb7f8712ad7d 2024-12-07T15:14:36,564 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/recovered.edits/502.seqid to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39/recovered.edits/502.seqid 2024-12-07T15:14:36,565 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/b393b7989f53c0069cd9db33cd3a7e39 2024-12-07T15:14:36,565 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-07T15:14:36,566 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-07T15:14:36,566 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-07T15:14:36,573 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412070134dacf0f68428685c7b670ff910c3f_b393b7989f53c0069cd9db33cd3a7e39 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412070134dacf0f68428685c7b670ff910c3f_b393b7989f53c0069cd9db33cd3a7e39 2024-12-07T15:14:36,574 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207056d1ade2a984ee2a09c328fd041a234_b393b7989f53c0069cd9db33cd3a7e39 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207056d1ade2a984ee2a09c328fd041a234_b393b7989f53c0069cd9db33cd3a7e39 2024-12-07T15:14:36,575 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207072ce1d4ea7c46c197c473596448b247_b393b7989f53c0069cd9db33cd3a7e39 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207072ce1d4ea7c46c197c473596448b247_b393b7989f53c0069cd9db33cd3a7e39 2024-12-07T15:14:36,577 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412070a1fab824a144234aabf25850171fdea_b393b7989f53c0069cd9db33cd3a7e39 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412070a1fab824a144234aabf25850171fdea_b393b7989f53c0069cd9db33cd3a7e39 2024-12-07T15:14:36,578 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412071835c78ba4904e32af9606a2cb39c1be_b393b7989f53c0069cd9db33cd3a7e39 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412071835c78ba4904e32af9606a2cb39c1be_b393b7989f53c0069cd9db33cd3a7e39 2024-12-07T15:14:36,579 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207189346f928b847d780fa433a5c15855f_b393b7989f53c0069cd9db33cd3a7e39 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207189346f928b847d780fa433a5c15855f_b393b7989f53c0069cd9db33cd3a7e39 2024-12-07T15:14:36,580 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120719364c1dcb064babb3b73d6fa722f533_b393b7989f53c0069cd9db33cd3a7e39 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120719364c1dcb064babb3b73d6fa722f533_b393b7989f53c0069cd9db33cd3a7e39 2024-12-07T15:14:36,582 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412072d7ab2765ff546048b30348287f609fb_b393b7989f53c0069cd9db33cd3a7e39 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412072d7ab2765ff546048b30348287f609fb_b393b7989f53c0069cd9db33cd3a7e39 2024-12-07T15:14:36,584 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120736d958b54ec7499b8904ee529e00e937_b393b7989f53c0069cd9db33cd3a7e39 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120736d958b54ec7499b8904ee529e00e937_b393b7989f53c0069cd9db33cd3a7e39 2024-12-07T15:14:36,585 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207498e5deb01564163b09708bdf681f447_b393b7989f53c0069cd9db33cd3a7e39 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207498e5deb01564163b09708bdf681f447_b393b7989f53c0069cd9db33cd3a7e39 2024-12-07T15:14:36,587 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120751f453b424754828802c5148b688d956_b393b7989f53c0069cd9db33cd3a7e39 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120751f453b424754828802c5148b688d956_b393b7989f53c0069cd9db33cd3a7e39 2024-12-07T15:14:36,589 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207657aec4b268f4761a72729173b828095_b393b7989f53c0069cd9db33cd3a7e39 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207657aec4b268f4761a72729173b828095_b393b7989f53c0069cd9db33cd3a7e39 2024-12-07T15:14:36,591 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412076a9c7617304d471282afacafc39897a5_b393b7989f53c0069cd9db33cd3a7e39 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412076a9c7617304d471282afacafc39897a5_b393b7989f53c0069cd9db33cd3a7e39 2024-12-07T15:14:36,592 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207835dc7cd66644d81aec4f28f08117fa4_b393b7989f53c0069cd9db33cd3a7e39 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207835dc7cd66644d81aec4f28f08117fa4_b393b7989f53c0069cd9db33cd3a7e39 2024-12-07T15:14:36,594 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207a55a25b7421e4f449d1a2f15094baefd_b393b7989f53c0069cd9db33cd3a7e39 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207a55a25b7421e4f449d1a2f15094baefd_b393b7989f53c0069cd9db33cd3a7e39 2024-12-07T15:14:36,595 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207c72a2af2820d48b7bd49ea7a036aa950_b393b7989f53c0069cd9db33cd3a7e39 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207c72a2af2820d48b7bd49ea7a036aa950_b393b7989f53c0069cd9db33cd3a7e39 2024-12-07T15:14:36,597 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207c8d9f5416b0a417e861bc3d42d4b0e18_b393b7989f53c0069cd9db33cd3a7e39 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207c8d9f5416b0a417e861bc3d42d4b0e18_b393b7989f53c0069cd9db33cd3a7e39 2024-12-07T15:14:36,602 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207ca766206f1e9469882f402b2fb03228e_b393b7989f53c0069cd9db33cd3a7e39 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207ca766206f1e9469882f402b2fb03228e_b393b7989f53c0069cd9db33cd3a7e39 2024-12-07T15:14:36,604 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207d2a151fb32ea4aec93ebfe647a0e3b75_b393b7989f53c0069cd9db33cd3a7e39 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207d2a151fb32ea4aec93ebfe647a0e3b75_b393b7989f53c0069cd9db33cd3a7e39 2024-12-07T15:14:36,605 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207eda85a80ba934844a3cacc61d63c1bed_b393b7989f53c0069cd9db33cd3a7e39 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207eda85a80ba934844a3cacc61d63c1bed_b393b7989f53c0069cd9db33cd3a7e39 2024-12-07T15:14:36,606 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207fd586cc427594221be0e4ac160ff1f41_b393b7989f53c0069cd9db33cd3a7e39 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207fd586cc427594221be0e4ac160ff1f41_b393b7989f53c0069cd9db33cd3a7e39 2024-12-07T15:14:36,607 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-07T15:14:36,610 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=58, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-07T15:14:36,613 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-07T15:14:36,615 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-07T15:14:36,617 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=58, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-07T15:14:36,617 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-07T15:14:36,617 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733584476617"}]},"ts":"9223372036854775807"} 2024-12-07T15:14:36,621 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-07T15:14:36,621 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => b393b7989f53c0069cd9db33cd3a7e39, NAME => 'TestAcidGuarantees,,1733584444709.b393b7989f53c0069cd9db33cd3a7e39.', STARTKEY => '', ENDKEY => ''}] 2024-12-07T15:14:36,621 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-07T15:14:36,621 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733584476621"}]},"ts":"9223372036854775807"} 2024-12-07T15:14:36,623 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-07T15:14:36,624 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=58, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-07T15:14:36,626 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 99 msec 2024-12-07T15:14:36,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-07T15:14:36,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-07T15:14:36,634 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-07T15:14:36,645 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=251 (was 248) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1222216727_22 at /127.0.0.1:60876 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5975031d-e054-e0fa-13a8-f8e344c3a20d/cluster_8a1d19e0-6a6c-4b1f-8318-b9e148680dc0/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5975031d-e054-e0fa-13a8-f8e344c3a20d/cluster_8a1d19e0-6a6c-4b1f-8318-b9e148680dc0/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=457 (was 453) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=433 (was 411) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7363 (was 7211) - AvailableMemoryMB LEAK? - 2024-12-07T15:14:36,658 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=251, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=433, ProcessCount=11, AvailableMemoryMB=7362 2024-12-07T15:14:36,659 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-07T15:14:36,660 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-07T15:14:36,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] procedure2.ProcedureExecutor(1139): Stored pid=59, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-07T15:14:36,662 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-07T15:14:36,662 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:36,663 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 59 2024-12-07T15:14:36,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-12-07T15:14:36,664 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-07T15:14:36,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742056_1232 (size=1061) 2024-12-07T15:14:36,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-12-07T15:14:36,775 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-07T15:14:36,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-12-07T15:14:37,072 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd 2024-12-07T15:14:37,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742057_1233 (size=53) 2024-12-07T15:14:37,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-12-07T15:14:37,479 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T15:14:37,479 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 3cb6bfad51268ddac87316028a55f638, disabling compactions & flushes 2024-12-07T15:14:37,479 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:37,479 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:37,479 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. after waiting 0 ms 2024-12-07T15:14:37,479 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:37,479 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:37,479 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 3cb6bfad51268ddac87316028a55f638: Waiting for close lock at 1733584477479Disabling compacts and flushes for region at 1733584477479Disabling writes for close at 1733584477479Writing region close event to WAL at 1733584477479Closed at 1733584477479 2024-12-07T15:14:37,480 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-07T15:14:37,480 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733584477480"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733584477480"}]},"ts":"1733584477480"} 2024-12-07T15:14:37,482 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-07T15:14:37,482 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-07T15:14:37,483 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733584477482"}]},"ts":"1733584477482"} 2024-12-07T15:14:37,484 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-07T15:14:37,484 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3cb6bfad51268ddac87316028a55f638, ASSIGN}] 2024-12-07T15:14:37,485 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3cb6bfad51268ddac87316028a55f638, ASSIGN 2024-12-07T15:14:37,486 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3cb6bfad51268ddac87316028a55f638, ASSIGN; state=OFFLINE, location=d6797ebe9c52,46009,1733584418401; forceNewPlan=false, retain=false 2024-12-07T15:14:37,636 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=3cb6bfad51268ddac87316028a55f638, regionState=OPENING, regionLocation=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:37,638 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3cb6bfad51268ddac87316028a55f638, ASSIGN because future has completed 2024-12-07T15:14:37,638 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401}] 2024-12-07T15:14:37,793 INFO [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:37,793 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7752): Opening region: {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} 2024-12-07T15:14:37,794 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 3cb6bfad51268ddac87316028a55f638 2024-12-07T15:14:37,794 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T15:14:37,794 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7794): checking encryption for 3cb6bfad51268ddac87316028a55f638 2024-12-07T15:14:37,794 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7797): checking classloading for 3cb6bfad51268ddac87316028a55f638 2024-12-07T15:14:37,795 INFO [StoreOpener-3cb6bfad51268ddac87316028a55f638-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 3cb6bfad51268ddac87316028a55f638 2024-12-07T15:14:37,796 INFO [StoreOpener-3cb6bfad51268ddac87316028a55f638-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-07T15:14:37,797 INFO [StoreOpener-3cb6bfad51268ddac87316028a55f638-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3cb6bfad51268ddac87316028a55f638 columnFamilyName A 2024-12-07T15:14:37,797 DEBUG [StoreOpener-3cb6bfad51268ddac87316028a55f638-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:37,797 INFO [StoreOpener-3cb6bfad51268ddac87316028a55f638-1 {}] regionserver.HStore(327): Store=3cb6bfad51268ddac87316028a55f638/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T15:14:37,798 INFO [StoreOpener-3cb6bfad51268ddac87316028a55f638-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 3cb6bfad51268ddac87316028a55f638 2024-12-07T15:14:37,799 INFO [StoreOpener-3cb6bfad51268ddac87316028a55f638-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-07T15:14:37,799 INFO [StoreOpener-3cb6bfad51268ddac87316028a55f638-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3cb6bfad51268ddac87316028a55f638 columnFamilyName B 2024-12-07T15:14:37,799 DEBUG [StoreOpener-3cb6bfad51268ddac87316028a55f638-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:37,799 INFO [StoreOpener-3cb6bfad51268ddac87316028a55f638-1 {}] regionserver.HStore(327): Store=3cb6bfad51268ddac87316028a55f638/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T15:14:37,800 INFO [StoreOpener-3cb6bfad51268ddac87316028a55f638-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 3cb6bfad51268ddac87316028a55f638 2024-12-07T15:14:37,801 INFO [StoreOpener-3cb6bfad51268ddac87316028a55f638-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-07T15:14:37,801 INFO [StoreOpener-3cb6bfad51268ddac87316028a55f638-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3cb6bfad51268ddac87316028a55f638 columnFamilyName C 2024-12-07T15:14:37,801 DEBUG [StoreOpener-3cb6bfad51268ddac87316028a55f638-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:14:37,801 INFO [StoreOpener-3cb6bfad51268ddac87316028a55f638-1 {}] regionserver.HStore(327): Store=3cb6bfad51268ddac87316028a55f638/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T15:14:37,801 INFO [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:37,801 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1038): replaying wal for 3cb6bfad51268ddac87316028a55f638 2024-12-07T15:14:37,802 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638 2024-12-07T15:14:37,802 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638 2024-12-07T15:14:37,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-12-07T15:14:37,804 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1048): stopping wal replay for 3cb6bfad51268ddac87316028a55f638 2024-12-07T15:14:37,804 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1060): Cleaning up temporary data for 3cb6bfad51268ddac87316028a55f638 2024-12-07T15:14:37,804 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-07T15:14:37,806 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1093): writing seq id for 3cb6bfad51268ddac87316028a55f638 2024-12-07T15:14:37,808 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T15:14:37,808 INFO [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1114): Opened 3cb6bfad51268ddac87316028a55f638; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59700767, jitterRate=-0.11038924753665924}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-07T15:14:37,808 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3cb6bfad51268ddac87316028a55f638 2024-12-07T15:14:37,810 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1006): Region open journal for 3cb6bfad51268ddac87316028a55f638: Running coprocessor pre-open hook at 1733584477794Writing region info on filesystem at 1733584477794Initializing all the Stores at 1733584477795 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733584477795Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733584477795Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733584477795Cleaning up temporary data from old regions at 1733584477804 (+9 ms)Running coprocessor post-open hooks at 1733584477808 (+4 ms)Region opened successfully at 1733584477810 (+2 ms) 2024-12-07T15:14:37,810 INFO [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., pid=61, masterSystemTime=1733584477790 2024-12-07T15:14:37,812 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:37,812 INFO [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:37,813 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=3cb6bfad51268ddac87316028a55f638, regionState=OPEN, openSeqNum=2, regionLocation=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:37,815 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 because future has completed 2024-12-07T15:14:37,817 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-12-07T15:14:37,817 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 in 177 msec 2024-12-07T15:14:37,819 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=59 2024-12-07T15:14:37,819 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=59, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3cb6bfad51268ddac87316028a55f638, ASSIGN in 333 msec 2024-12-07T15:14:37,819 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-07T15:14:37,819 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733584477819"}]},"ts":"1733584477819"} 2024-12-07T15:14:37,821 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-07T15:14:37,822 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-07T15:14:37,823 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1620 sec 2024-12-07T15:14:38,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-12-07T15:14:38,813 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-07T15:14:38,815 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e04d2b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:14:38,815 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6797ebe9c52,36915,-1 for getting cluster id 2024-12-07T15:14:38,815 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T15:14:38,816 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f8956274-ed33-4e16-82eb-d5d5911ec893' 2024-12-07T15:14:38,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T15:14:38,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f8956274-ed33-4e16-82eb-d5d5911ec893" 2024-12-07T15:14:38,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@214ba631, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:14:38,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6797ebe9c52,36915,-1] 2024-12-07T15:14:38,818 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T15:14:38,818 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:14:38,819 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58642, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T15:14:38,819 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@867404c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:14:38,820 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T15:14:38,821 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6797ebe9c52,46009,1733584418401, seqNum=-1] 2024-12-07T15:14:38,821 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T15:14:38,822 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40084, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T15:14:38,823 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-07T15:14:38,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is d6797ebe9c52,36915,1733584417704 2024-12-07T15:14:38,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@69bc622c 2024-12-07T15:14:38,825 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-07T15:14:38,826 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58648, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-07T15:14:38,827 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@391cff66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:14:38,827 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6797ebe9c52,36915,-1 for getting cluster id 2024-12-07T15:14:38,827 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T15:14:38,828 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f8956274-ed33-4e16-82eb-d5d5911ec893' 2024-12-07T15:14:38,828 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T15:14:38,828 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f8956274-ed33-4e16-82eb-d5d5911ec893" 2024-12-07T15:14:38,829 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10aeb4b2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:14:38,829 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6797ebe9c52,36915,-1] 2024-12-07T15:14:38,829 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T15:14:38,829 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:14:38,830 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58670, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T15:14:38,830 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32401964, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:14:38,831 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@135bd0ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:14:38,831 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6797ebe9c52,36915,-1 for getting cluster id 2024-12-07T15:14:38,831 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T15:14:38,832 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f8956274-ed33-4e16-82eb-d5d5911ec893' 2024-12-07T15:14:38,832 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T15:14:38,832 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f8956274-ed33-4e16-82eb-d5d5911ec893" 2024-12-07T15:14:38,833 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77963136, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:14:38,833 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6797ebe9c52,36915,-1] 2024-12-07T15:14:38,833 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T15:14:38,833 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:14:38,834 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58692, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T15:14:38,835 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a02f49c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:14:38,835 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4609cead, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:14:38,835 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6797ebe9c52,36915,-1 for getting cluster id 2024-12-07T15:14:38,836 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T15:14:38,840 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f8956274-ed33-4e16-82eb-d5d5911ec893' 2024-12-07T15:14:38,840 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T15:14:38,840 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f8956274-ed33-4e16-82eb-d5d5911ec893" 2024-12-07T15:14:38,840 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e3607fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:14:38,840 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6797ebe9c52,36915,-1] 2024-12-07T15:14:38,840 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T15:14:38,841 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:14:38,841 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58718, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T15:14:38,842 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56947e58, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:14:38,844 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a2df097, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:14:38,844 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6797ebe9c52,36915,-1 for getting cluster id 2024-12-07T15:14:38,845 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T15:14:38,848 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f8956274-ed33-4e16-82eb-d5d5911ec893' 2024-12-07T15:14:38,849 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T15:14:38,849 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f8956274-ed33-4e16-82eb-d5d5911ec893" 2024-12-07T15:14:38,849 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@550d7d37, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:14:38,849 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6797ebe9c52,36915,-1] 2024-12-07T15:14:38,849 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T15:14:38,849 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:14:38,850 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58734, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T15:14:38,851 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c6e0e3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:14:38,852 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2856abc4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:14:38,853 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6797ebe9c52,36915,-1 for getting cluster id 2024-12-07T15:14:38,853 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T15:14:38,853 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f8956274-ed33-4e16-82eb-d5d5911ec893' 2024-12-07T15:14:38,854 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T15:14:38,854 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f8956274-ed33-4e16-82eb-d5d5911ec893" 2024-12-07T15:14:38,855 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5eeec3cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:14:38,855 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6797ebe9c52,36915,-1] 2024-12-07T15:14:38,855 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T15:14:38,855 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:14:38,856 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58740, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T15:14:38,857 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e8b3700, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:14:38,859 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a4f6020, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:14:38,859 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6797ebe9c52,36915,-1 for getting cluster id 2024-12-07T15:14:38,859 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T15:14:38,861 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f8956274-ed33-4e16-82eb-d5d5911ec893' 2024-12-07T15:14:38,861 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T15:14:38,861 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f8956274-ed33-4e16-82eb-d5d5911ec893" 2024-12-07T15:14:38,861 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5537eda6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:14:38,861 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6797ebe9c52,36915,-1] 2024-12-07T15:14:38,862 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T15:14:38,862 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:14:38,863 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58750, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T15:14:38,864 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@669e284, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:14:38,864 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@692cba37, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:14:38,864 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6797ebe9c52,36915,-1 for getting cluster id 2024-12-07T15:14:38,865 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T15:14:38,865 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f8956274-ed33-4e16-82eb-d5d5911ec893' 2024-12-07T15:14:38,865 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T15:14:38,865 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f8956274-ed33-4e16-82eb-d5d5911ec893" 2024-12-07T15:14:38,865 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d34045, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:14:38,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6797ebe9c52,36915,-1] 2024-12-07T15:14:38,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T15:14:38,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:14:38,866 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58772, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T15:14:38,867 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e8f0e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:14:38,868 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d805c88, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:14:38,868 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6797ebe9c52,36915,-1 for getting cluster id 2024-12-07T15:14:38,868 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T15:14:38,869 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f8956274-ed33-4e16-82eb-d5d5911ec893' 2024-12-07T15:14:38,869 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T15:14:38,869 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f8956274-ed33-4e16-82eb-d5d5911ec893" 2024-12-07T15:14:38,869 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53bdf5a8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:14:38,869 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6797ebe9c52,36915,-1] 2024-12-07T15:14:38,869 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T15:14:38,869 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:14:38,870 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58788, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T15:14:38,870 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e4ca728, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:14:38,871 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4861c2d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:14:38,871 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6797ebe9c52,36915,-1 for getting cluster id 2024-12-07T15:14:38,871 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T15:14:38,872 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f8956274-ed33-4e16-82eb-d5d5911ec893' 2024-12-07T15:14:38,872 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T15:14:38,872 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f8956274-ed33-4e16-82eb-d5d5911ec893" 2024-12-07T15:14:38,872 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f38ac1e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:14:38,872 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6797ebe9c52,36915,-1] 2024-12-07T15:14:38,872 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T15:14:38,872 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:14:38,874 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58804, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T15:14:38,874 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65138cca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:14:38,875 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3397c776, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:14:38,875 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6797ebe9c52,36915,-1 for getting cluster id 2024-12-07T15:14:38,875 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T15:14:38,876 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f8956274-ed33-4e16-82eb-d5d5911ec893' 2024-12-07T15:14:38,876 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T15:14:38,876 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f8956274-ed33-4e16-82eb-d5d5911ec893" 2024-12-07T15:14:38,876 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b896ed5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:14:38,876 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6797ebe9c52,36915,-1] 2024-12-07T15:14:38,877 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T15:14:38,877 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:14:38,877 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58826, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T15:14:38,878 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ccbdf24, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:14:38,879 DEBUG [Thread-1097 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T15:14:38,879 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6797ebe9c52,46009,1733584418401, seqNum=-1] 2024-12-07T15:14:38,880 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T15:14:38,881 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40098, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T15:14:38,882 DEBUG [Thread-1108 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T15:14:38,883 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2] 2024-12-07T15:14:38,883 DEBUG [Thread-1102 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T15:14:38,884 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6797ebe9c52,46009,1733584418401, seqNum=-1] 2024-12-07T15:14:38,884 DEBUG [Thread-1108 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6797ebe9c52,46009,1733584418401, seqNum=-1] 2024-12-07T15:14:38,884 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T15:14:38,884 DEBUG [Thread-1108 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T15:14:38,885 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40108, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T15:14:38,885 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40100, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T15:14:38,888 DEBUG [Thread-1108 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2] 2024-12-07T15:14:38,888 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2] 2024-12-07T15:14:38,894 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T15:14:38,895 DEBUG [Thread-1100 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T15:14:38,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-12-07T15:14:38,896 DEBUG [Thread-1106 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T15:14:38,896 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T15:14:38,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-07T15:14:38,896 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6797ebe9c52,46009,1733584418401, seqNum=-1] 2024-12-07T15:14:38,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T15:14:38,897 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T15:14:38,897 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T15:14:38,897 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6797ebe9c52,46009,1733584418401, seqNum=-1] 2024-12-07T15:14:38,898 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T15:14:38,898 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40114, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T15:14:38,899 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40126, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T15:14:38,900 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2] 2024-12-07T15:14:38,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2] 2024-12-07T15:14:38,902 DEBUG [Thread-1104 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T15:14:38,902 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6797ebe9c52,46009,1733584418401, seqNum=-1] 2024-12-07T15:14:38,903 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T15:14:38,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on 3cb6bfad51268ddac87316028a55f638 2024-12-07T15:14:38,904 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3cb6bfad51268ddac87316028a55f638 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-07T15:14:38,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3cb6bfad51268ddac87316028a55f638, store=A 2024-12-07T15:14:38,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:14:38,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3cb6bfad51268ddac87316028a55f638, store=B 2024-12-07T15:14:38,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:14:38,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3cb6bfad51268ddac87316028a55f638, store=C 2024-12-07T15:14:38,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:14:38,907 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40142, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T15:14:38,909 DEBUG [Thread-1091 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T15:14:38,909 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2] 2024-12-07T15:14:38,909 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6797ebe9c52,46009,1733584418401, seqNum=-1] 2024-12-07T15:14:38,910 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T15:14:38,911 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40150, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T15:14:38,913 DEBUG [Thread-1089 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T15:14:38,913 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2] 2024-12-07T15:14:38,913 DEBUG [Thread-1089 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6797ebe9c52,46009,1733584418401, seqNum=-1] 2024-12-07T15:14:38,914 DEBUG [Thread-1089 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T15:14:38,914 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40166, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T15:14:38,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2] 2024-12-07T15:14:38,933 DEBUG [Thread-1093 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T15:14:38,934 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6797ebe9c52,46009,1733584418401, seqNum=-1] 2024-12-07T15:14:38,934 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T15:14:38,935 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40170, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T15:14:38,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2] 2024-12-07T15:14:38,946 DEBUG [Thread-1095 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T15:14:38,947 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6797ebe9c52,46009,1733584418401, seqNum=-1] 2024-12-07T15:14:38,948 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T15:14:38,949 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40182, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T15:14:38,951 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2] 2024-12-07T15:14:38,957 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/A/b419ed8660bf411d884e1c34ac8dab96 is 50, key is test_row_0/A:col10/1733584478901/Put/seqid=0 2024-12-07T15:14:38,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742058_1234 (size=12001) 2024-12-07T15:14:38,985 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:38,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40170 deadline: 1733584538978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:38,986 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:38,986 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:38,986 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:38,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:38,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40098 deadline: 1733584538981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:38,987 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733584478967, max timestamp=1733584478983Num uniques 859; 2024-12-07T15:14:38,987 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-07T15:14:38,988 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:38,988 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:38,988 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:38,990 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733584478967, max timestamp=1733584478984Num uniques 859; 2024-12-07T15:14:38,990 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-07T15:14:38,995 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733584478967, max timestamp=1733584478984Num uniques 859; 2024-12-07T15:14:38,995 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-07T15:14:39,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-07T15:14:39,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:39,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40150 deadline: 1733584539026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:39,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:39,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:39,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:39,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:39,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40182 deadline: 1733584539030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:39,035 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:39,035 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:39,035 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:39,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:39,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40166 deadline: 1733584539031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:39,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:39,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:39,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:39,051 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-07T15:14:39,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:39,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. as already flushing 2024-12-07T15:14:39,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:39,052 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:39,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:39,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:39,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:39,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40170 deadline: 1733584539093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:39,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:39,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:39,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:39,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:39,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40098 deadline: 1733584539093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:39,099 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:39,099 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:39,099 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:39,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:39,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40150 deadline: 1733584539143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:39,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:39,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:39,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40182 deadline: 1733584539143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:39,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:39,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:39,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40166 deadline: 1733584539143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:39,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:39,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:39,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:39,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:39,149 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:39,149 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:39,149 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:39,206 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-07T15:14:39,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:39,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. as already flushing 2024-12-07T15:14:39,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:39,206 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:39,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:39,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:39,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-07T15:14:39,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:39,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40170 deadline: 1733584539303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:39,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:39,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:39,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40098 deadline: 1733584539303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:39,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:39,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:39,307 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:39,307 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:39,307 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:39,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:39,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40166 deadline: 1733584539354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:39,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:39,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:39,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40150 deadline: 1733584539354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:39,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:39,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:39,356 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:39,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:39,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:39,358 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:39,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40182 deadline: 1733584539354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:39,359 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:39,359 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:39,359 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:39,360 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-07T15:14:39,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:39,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. as already flushing 2024-12-07T15:14:39,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:39,360 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:39,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:39,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:39,379 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/A/b419ed8660bf411d884e1c34ac8dab96 2024-12-07T15:14:39,419 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/B/d3b97759282e41e78d6e2be2d0d29067 is 50, key is test_row_0/B:col10/1733584478901/Put/seqid=0 2024-12-07T15:14:39,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742059_1235 (size=12001) 2024-12-07T15:14:39,514 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-07T15:14:39,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:39,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. as already flushing 2024-12-07T15:14:39,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:39,514 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:39,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:39,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:39,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-07T15:14:39,617 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:39,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40170 deadline: 1733584539614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:39,617 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:39,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40098 deadline: 1733584539614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:39,618 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:39,618 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:39,618 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:39,618 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:39,618 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:39,618 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:39,673 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-07T15:14:39,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:39,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. as already flushing 2024-12-07T15:14:39,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:39,674 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:39,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:39,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:39,676 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:39,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40166 deadline: 1733584539672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:39,676 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:39,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40150 deadline: 1733584539673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:39,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:39,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40182 deadline: 1733584539673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:39,677 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:39,677 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:39,677 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:39,677 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:39,677 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:39,677 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:39,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:39,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:39,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:39,832 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-07T15:14:39,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:39,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. as already flushing 2024-12-07T15:14:39,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:39,833 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:39,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:39,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:39,870 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/B/d3b97759282e41e78d6e2be2d0d29067 2024-12-07T15:14:39,960 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/C/cb9b93545cfe4ce38cce242439564f52 is 50, key is test_row_0/C:col10/1733584478901/Put/seqid=0 2024-12-07T15:14:39,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742060_1236 (size=12001) 2024-12-07T15:14:39,988 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-07T15:14:39,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:39,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. as already flushing 2024-12-07T15:14:39,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:39,988 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:39,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:39,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:39,990 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/C/cb9b93545cfe4ce38cce242439564f52 2024-12-07T15:14:39,997 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/A/b419ed8660bf411d884e1c34ac8dab96 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/b419ed8660bf411d884e1c34ac8dab96 2024-12-07T15:14:40,006 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/b419ed8660bf411d884e1c34ac8dab96, entries=150, sequenceid=11, filesize=11.7 K 2024-12-07T15:14:40,008 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/B/d3b97759282e41e78d6e2be2d0d29067 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/d3b97759282e41e78d6e2be2d0d29067 2024-12-07T15:14:40,014 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/d3b97759282e41e78d6e2be2d0d29067, entries=150, sequenceid=11, filesize=11.7 K 2024-12-07T15:14:40,015 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/C/cb9b93545cfe4ce38cce242439564f52 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/cb9b93545cfe4ce38cce242439564f52 2024-12-07T15:14:40,020 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/cb9b93545cfe4ce38cce242439564f52, entries=150, sequenceid=11, filesize=11.7 K 2024-12-07T15:14:40,022 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=133.39 KB/136590 for 3cb6bfad51268ddac87316028a55f638 in 1117ms, sequenceid=11, compaction requested=false 2024-12-07T15:14:40,022 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3cb6bfad51268ddac87316028a55f638: 2024-12-07T15:14:40,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-07T15:14:40,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on 3cb6bfad51268ddac87316028a55f638 2024-12-07T15:14:40,141 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-07T15:14:40,141 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733584479020, max timestamp=1733584480136Num uniques 1009; 2024-12-07T15:14:40,141 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733584479020, max timestamp=1733584480136Num uniques 1018; 2024-12-07T15:14:40,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:40,141 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=7688, suffix heap size=132904,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=168 2024-12-07T15:14:40,141 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-07T15:14:40,141 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing 3cb6bfad51268ddac87316028a55f638 3/3 column families, dataSize=64.46 KB heapSize=177.88 KB 2024-12-07T15:14:40,141 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. as already flushing 2024-12-07T15:14:40,141 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733584479020, max timestamp=1733584480136Num uniques 1009; 2024-12-07T15:14:40,141 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=7688, suffix heap size=132904,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=168 2024-12-07T15:14:40,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3cb6bfad51268ddac87316028a55f638, store=A 2024-12-07T15:14:40,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T15:14:40,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3cb6bfad51268ddac87316028a55f638, store=B 2024-12-07T15:14:40,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T15:14:40,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3cb6bfad51268ddac87316028a55f638, store=C 2024-12-07T15:14:40,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T15:14:40,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/A/1613379d99c64ecdb95a3592025a1cd3 is 50, key is test_row_0/A:col10/1733584479029/Put/seqid=0 2024-12-07T15:14:40,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742061_1237 (size=14341) 2024-12-07T15:14:40,214 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733584480196, max timestamp=1733584480208Num uniques 859; 2024-12-07T15:14:40,214 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-07T15:14:40,214 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733584480196, max timestamp=1733584480208Num uniques 859; 2024-12-07T15:14:40,214 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-07T15:14:40,214 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733584480196, max timestamp=1733584480208Num uniques 859; 2024-12-07T15:14:40,214 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-07T15:14:40,252 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733584480235, max timestamp=1733584480248Num uniques 1068; 2024-12-07T15:14:40,252 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-12-07T15:14:40,253 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7681, totalHeapSize=17712, min timestamp=1733584480235, max timestamp=1733584480248Num uniques 1068; 2024-12-07T15:14:40,253 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48901, new segment data size=7681, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-12-07T15:14:40,256 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733584480235, max timestamp=1733584480248Num uniques 1068; 2024-12-07T15:14:40,256 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=6870, suffix heap size=139040,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=150 2024-12-07T15:14:40,294 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733584480277, max timestamp=1733584480290Num uniques 1027; 2024-12-07T15:14:40,294 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-12-07T15:14:40,295 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733584480277, max timestamp=1733584480290Num uniques 1027; 2024-12-07T15:14:40,295 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-12-07T15:14:40,296 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733584480277, max timestamp=1733584480290Num uniques 1009; 2024-12-07T15:14:40,296 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=8102, suffix heap size=132904,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=177 2024-12-07T15:14:40,332 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1733584480308, max timestamp=1733584480327Num uniques 1036; 2024-12-07T15:14:40,332 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-12-07T15:14:40,332 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1733584480308, max timestamp=1733584480327Num uniques 1036; 2024-12-07T15:14:40,332 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-12-07T15:14:40,335 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733584480308, max timestamp=1733584480327Num uniques 1036; 2024-12-07T15:14:40,335 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-12-07T15:14:40,371 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1733584480346, max timestamp=1733584480365Num uniques 1095; 2024-12-07T15:14:40,372 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=13510, suffix heap size=141848,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=295 2024-12-07T15:14:40,372 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1733584480346, max timestamp=1733584480365Num uniques 1095; 2024-12-07T15:14:40,372 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=13510, suffix heap size=141848,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=295 2024-12-07T15:14:40,376 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733584480360, max timestamp=1733584480365Num uniques 1045; 2024-12-07T15:14:40,376 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=6870, suffix heap size=136648,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=150 2024-12-07T15:14:40,412 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1733584480386, max timestamp=1733584480407Num uniques 1154; 2024-12-07T15:14:40,412 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=52850, new segment data size=9340, suffix heap size=147984,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1154, new segment cells count=204 2024-12-07T15:14:40,412 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1733584480386, max timestamp=1733584480407Num uniques 1154; 2024-12-07T15:14:40,412 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1733584480386, max timestamp=1733584480407Num uniques 1009; 2024-12-07T15:14:40,413 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=52850, new segment data size=9340, suffix heap size=147984,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1154, new segment cells count=204 2024-12-07T15:14:40,413 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46210, new segment data size=9340, suffix heap size=132904,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=204 2024-12-07T15:14:40,449 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1733584480433, max timestamp=1733584480444Num uniques 1063; 2024-12-07T15:14:40,449 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1733584480433, max timestamp=1733584480444Num uniques 1063; 2024-12-07T15:14:40,449 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=9748, suffix heap size=138520,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=213 2024-12-07T15:14:40,449 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=9748, suffix heap size=138520,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=213 2024-12-07T15:14:40,449 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1733584480433, max timestamp=1733584480444Num uniques 1063; 2024-12-07T15:14:40,449 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=9748, suffix heap size=138520,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=213 2024-12-07T15:14:40,485 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1733584480467, max timestamp=1733584480480Num uniques 1072; 2024-12-07T15:14:40,485 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=10162, suffix heap size=139456,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=222 2024-12-07T15:14:40,485 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1733584480467, max timestamp=1733584480480Num uniques 1072; 2024-12-07T15:14:40,485 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=10162, suffix heap size=139456,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=222 2024-12-07T15:14:40,489 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1733584480467, max timestamp=1733584480480Num uniques 1072; 2024-12-07T15:14:40,489 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=7872, suffix heap size=139456,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=172 2024-12-07T15:14:40,526 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1733584480507, max timestamp=1733584480521Num uniques 1081; 2024-12-07T15:14:40,526 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49506, new segment data size=10576, suffix heap size=140392,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=231 2024-12-07T15:14:40,526 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1733584480507, max timestamp=1733584480521Num uniques 1081; 2024-12-07T15:14:40,526 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49506, new segment data size=10576, suffix heap size=140392,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=231 2024-12-07T15:14:40,526 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1733584480507, max timestamp=1733584480521Num uniques 1031; 2024-12-07T15:14:40,526 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=10576, suffix heap size=135192,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=231 2024-12-07T15:14:40,559 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=290, cellSize=13280, totalHeapSize=30400, min timestamp=1733584480533, max timestamp=1733584480553Num uniques 1090; 2024-12-07T15:14:40,559 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=13280, suffix heap size=141328,new segment heap size=30400 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=290 2024-12-07T15:14:40,563 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1733584480533, max timestamp=1733584480553Num uniques 1090; 2024-12-07T15:14:40,563 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=8700, suffix heap size=141328,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=190 2024-12-07T15:14:40,564 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1733584480533, max timestamp=1733584480559Num uniques 1090; 2024-12-07T15:14:40,564 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=8700, suffix heap size=141328,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=190 2024-12-07T15:14:40,587 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/A/1613379d99c64ecdb95a3592025a1cd3 2024-12-07T15:14:40,594 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=249, cellSize=11404, totalHeapSize=26136, min timestamp=1733584480578, max timestamp=1733584480589Num uniques 1149; 2024-12-07T15:14:40,595 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=52624, new segment data size=11404, suffix heap size=147464,new segment heap size=26136 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1149, new segment cells count=249 2024-12-07T15:14:40,598 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9114, totalHeapSize=20936, min timestamp=1733584480585, max timestamp=1733584480589Num uniques 1049; 2024-12-07T15:14:40,598 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=9114, suffix heap size=137064,new segment heap size=20936 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=199 2024-12-07T15:14:40,600 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733584480585, max timestamp=1733584480589Num uniques 1049; 2024-12-07T15:14:40,600 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=6870, suffix heap size=137064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=150 2024-12-07T15:14:40,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/B/4e0b7591f37b4ef8b7cf6eebd164e98d is 50, key is test_row_0/B:col10/1733584479029/Put/seqid=0 2024-12-07T15:14:40,630 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=208, cellSize=9520, totalHeapSize=21872, min timestamp=1733584480608, max timestamp=1733584480625Num uniques 1058; 2024-12-07T15:14:40,630 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48450, new segment data size=9520, suffix heap size=138000,new segment heap size=21872 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1058, new segment cells count=208 2024-12-07T15:14:40,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742062_1238 (size=12001) 2024-12-07T15:14:40,635 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/B/4e0b7591f37b4ef8b7cf6eebd164e98d 2024-12-07T15:14:40,638 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=208, cellSize=9520, totalHeapSize=21872, min timestamp=1733584480608, max timestamp=1733584480625Num uniques 1108; 2024-12-07T15:14:40,639 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=50740, new segment data size=9520, suffix heap size=143200,new segment heap size=21872 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1108, new segment cells count=208 2024-12-07T15:14:40,642 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=208, cellSize=9520, totalHeapSize=21872, min timestamp=1733584480608, max timestamp=1733584480625Num uniques 1009; 2024-12-07T15:14:40,642 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46206, new segment data size=9520, suffix heap size=132904,new segment heap size=21872 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=208 2024-12-07T15:14:40,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/C/f67e7ab9b95d430dbce9903449cb4c47 is 50, key is test_row_0/C:col10/1733584479029/Put/seqid=0 2024-12-07T15:14:40,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742063_1239 (size=12001) 2024-12-07T15:14:40,675 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/C/f67e7ab9b95d430dbce9903449cb4c47 2024-12-07T15:14:40,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/A/1613379d99c64ecdb95a3592025a1cd3 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/1613379d99c64ecdb95a3592025a1cd3 2024-12-07T15:14:40,688 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=217, cellSize=9932, totalHeapSize=22808, min timestamp=1733584480672, max timestamp=1733584480684Num uniques 1067; 2024-12-07T15:14:40,688 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48862, new segment data size=9932, suffix heap size=138936,new segment heap size=22808 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1067, new segment cells count=217 2024-12-07T15:14:40,688 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=217, cellSize=9932, totalHeapSize=22808, min timestamp=1733584480672, max timestamp=1733584480684Num uniques 1067; 2024-12-07T15:14:40,689 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48862, new segment data size=9932, suffix heap size=138936,new segment heap size=22808 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1067, new segment cells count=217 2024-12-07T15:14:40,691 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=217, cellSize=9932, totalHeapSize=22808, min timestamp=1733584480672, max timestamp=1733584480684Num uniques 1067; 2024-12-07T15:14:40,691 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48862, new segment data size=9932, suffix heap size=138936,new segment heap size=22808 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1067, new segment cells count=217 2024-12-07T15:14:40,694 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/1613379d99c64ecdb95a3592025a1cd3, entries=200, sequenceid=50, filesize=14.0 K 2024-12-07T15:14:40,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/B/4e0b7591f37b4ef8b7cf6eebd164e98d as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/4e0b7591f37b4ef8b7cf6eebd164e98d 2024-12-07T15:14:40,702 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/4e0b7591f37b4ef8b7cf6eebd164e98d, entries=150, sequenceid=50, filesize=11.7 K 2024-12-07T15:14:40,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/C/f67e7ab9b95d430dbce9903449cb4c47 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/f67e7ab9b95d430dbce9903449cb4c47 2024-12-07T15:14:40,711 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/f67e7ab9b95d430dbce9903449cb4c47, entries=150, sequenceid=50, filesize=11.7 K 2024-12-07T15:14:40,713 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~85.27 KB/87312, currentSize=120.76 KB/123660 for 3cb6bfad51268ddac87316028a55f638 in 572ms, sequenceid=50, compaction requested=false 2024-12-07T15:14:40,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for 3cb6bfad51268ddac87316028a55f638: 2024-12-07T15:14:40,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:40,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-12-07T15:14:40,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-12-07T15:14:40,717 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-12-07T15:14:40,717 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8180 sec 2024-12-07T15:14:40,720 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3cb6bfad51268ddac87316028a55f638 3/3 column families, dataSize=127.47 KB heapSize=359.63 KB 2024-12-07T15:14:40,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on 3cb6bfad51268ddac87316028a55f638 2024-12-07T15:14:40,721 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 1.8240 sec 2024-12-07T15:14:40,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3cb6bfad51268ddac87316028a55f638, store=A 2024-12-07T15:14:40,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T15:14:40,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3cb6bfad51268ddac87316028a55f638, store=B 2024-12-07T15:14:40,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T15:14:40,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3cb6bfad51268ddac87316028a55f638, store=C 2024-12-07T15:14:40,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T15:14:40,729 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/A/474a97542ab447028f9bf47484116cb0 is 50, key is test_row_0/A:col10/1733584480718/Put/seqid=0 2024-12-07T15:14:40,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:40,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40150 deadline: 1733584540726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:40,736 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:40,736 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:40,736 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:40,741 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:40,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40098 deadline: 1733584540730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:40,741 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:40,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40170 deadline: 1733584540731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:40,741 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:40,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40166 deadline: 1733584540733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:40,741 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:40,742 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:40,742 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:40,742 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:40,742 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:40,742 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:40,742 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:40,742 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:40,742 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:40,742 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:40,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40182 deadline: 1733584540736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:40,743 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:40,743 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:40,743 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:40,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742064_1240 (size=14741) 2024-12-07T15:14:40,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:40,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40166 deadline: 1733584540843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:40,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:40,849 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:40,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40150 deadline: 1733584540843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:40,849 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:40,849 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:40,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:40,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:40,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:40,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:40,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40170 deadline: 1733584540854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:40,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:40,858 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:40,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40098 deadline: 1733584540854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:40,858 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:40,858 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:40,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:40,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40182 deadline: 1733584540854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:40,858 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:40,858 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:40,858 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:40,859 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:40,859 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:40,859 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:41,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-07T15:14:41,044 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T15:14:41,044 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T15:14:41,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-12-07T15:14:41,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-07T15:14:41,047 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T15:14:41,048 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T15:14:41,048 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T15:14:41,059 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:41,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40150 deadline: 1733584541053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:41,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:41,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:41,060 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:41,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:41,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40166 deadline: 1733584541053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:41,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:41,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:41,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:41,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:41,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40170 deadline: 1733584541063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:41,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:41,066 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:41,066 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:41,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40098 deadline: 1733584541063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:41,066 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:41,067 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:41,067 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:41,067 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:41,068 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:41,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40182 deadline: 1733584541063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:41,069 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:41,069 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:41,069 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:41,150 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/A/474a97542ab447028f9bf47484116cb0 2024-12-07T15:14:41,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-07T15:14:41,164 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/B/9931ec432ead44278cb4b298f80e6c2e is 50, key is test_row_0/B:col10/1733584480718/Put/seqid=0 2024-12-07T15:14:41,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742065_1241 (size=12301) 2024-12-07T15:14:41,201 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-07T15:14:41,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:41,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. as already flushing 2024-12-07T15:14:41,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:41,202 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:41,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:41,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:41,355 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-07T15:14:41,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:41,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. as already flushing 2024-12-07T15:14:41,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:41,355 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:41,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:41,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:41,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-07T15:14:41,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:41,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40166 deadline: 1733584541363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:41,367 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:41,367 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:41,367 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:41,377 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:41,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40150 deadline: 1733584541373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:41,378 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:41,378 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:41,378 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:41,378 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:41,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40098 deadline: 1733584541374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:41,378 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:41,379 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:41,379 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:41,379 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:41,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40170 deadline: 1733584541374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:41,379 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:41,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:41,379 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:41,379 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:41,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40182 deadline: 1733584541374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:41,380 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:41,380 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:41,380 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:41,438 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-07T15:14:41,508 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-07T15:14:41,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:41,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. as already flushing 2024-12-07T15:14:41,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:41,509 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:41,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:41,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:41,573 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/B/9931ec432ead44278cb4b298f80e6c2e 2024-12-07T15:14:41,591 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/C/684491ae329b4ad79ea7f6b08d670f80 is 50, key is test_row_0/C:col10/1733584480718/Put/seqid=0 2024-12-07T15:14:41,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742066_1242 (size=12301) 2024-12-07T15:14:41,607 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/C/684491ae329b4ad79ea7f6b08d670f80 2024-12-07T15:14:41,617 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/A/474a97542ab447028f9bf47484116cb0 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/474a97542ab447028f9bf47484116cb0 2024-12-07T15:14:41,624 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/474a97542ab447028f9bf47484116cb0, entries=200, sequenceid=292, filesize=14.4 K 2024-12-07T15:14:41,626 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/B/9931ec432ead44278cb4b298f80e6c2e as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/9931ec432ead44278cb4b298f80e6c2e 2024-12-07T15:14:41,632 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/9931ec432ead44278cb4b298f80e6c2e, entries=150, sequenceid=292, filesize=12.0 K 2024-12-07T15:14:41,635 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/C/684491ae329b4ad79ea7f6b08d670f80 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/684491ae329b4ad79ea7f6b08d670f80 2024-12-07T15:14:41,639 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/684491ae329b4ad79ea7f6b08d670f80, entries=150, sequenceid=292, filesize=12.0 K 2024-12-07T15:14:41,640 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~379.50 KB/388608, currentSize=46.96 KB/48090 for 3cb6bfad51268ddac87316028a55f638 in 920ms, sequenceid=292, compaction requested=true 2024-12-07T15:14:41,640 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3cb6bfad51268ddac87316028a55f638: 2024-12-07T15:14:41,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3cb6bfad51268ddac87316028a55f638:A, priority=-2147483648, current under compaction store size is 1 2024-12-07T15:14:41,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:14:41,641 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:14:41,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3cb6bfad51268ddac87316028a55f638:B, priority=-2147483648, current under compaction store size is 2 2024-12-07T15:14:41,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T15:14:41,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3cb6bfad51268ddac87316028a55f638:C, priority=-2147483648, current under compaction store size is 3 2024-12-07T15:14:41,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-07T15:14:41,641 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:14:41,643 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41083 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:14:41,644 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1541): 3cb6bfad51268ddac87316028a55f638/A is initiating minor compaction (all files) 2024-12-07T15:14:41,644 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3cb6bfad51268ddac87316028a55f638/A in TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:41,644 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/b419ed8660bf411d884e1c34ac8dab96, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/1613379d99c64ecdb95a3592025a1cd3, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/474a97542ab447028f9bf47484116cb0] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp, totalSize=40.1 K 2024-12-07T15:14:41,645 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36303 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:14:41,645 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1541): 3cb6bfad51268ddac87316028a55f638/B is initiating minor compaction (all files) 2024-12-07T15:14:41,645 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3cb6bfad51268ddac87316028a55f638/B in TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:41,645 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/d3b97759282e41e78d6e2be2d0d29067, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/4e0b7591f37b4ef8b7cf6eebd164e98d, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/9931ec432ead44278cb4b298f80e6c2e] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp, totalSize=35.5 K 2024-12-07T15:14:41,646 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting b419ed8660bf411d884e1c34ac8dab96, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733584478886 2024-12-07T15:14:41,646 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting d3b97759282e41e78d6e2be2d0d29067, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733584478886 2024-12-07T15:14:41,646 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1613379d99c64ecdb95a3592025a1cd3, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733584479026 2024-12-07T15:14:41,647 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e0b7591f37b4ef8b7cf6eebd164e98d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733584479026 2024-12-07T15:14:41,647 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 474a97542ab447028f9bf47484116cb0, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1733584480708 2024-12-07T15:14:41,648 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 9931ec432ead44278cb4b298f80e6c2e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1733584480710 2024-12-07T15:14:41,662 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-07T15:14:41,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:41,663 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing 3cb6bfad51268ddac87316028a55f638 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-07T15:14:41,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3cb6bfad51268ddac87316028a55f638, store=A 2024-12-07T15:14:41,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:14:41,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3cb6bfad51268ddac87316028a55f638, store=B 2024-12-07T15:14:41,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:14:41,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3cb6bfad51268ddac87316028a55f638, store=C 2024-12-07T15:14:41,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:14:41,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-07T15:14:41,674 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3cb6bfad51268ddac87316028a55f638#A#compaction#197 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:14:41,675 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/A/5d14fc84f2c440c2abe9f767050c001d is 50, key is test_row_0/A:col10/1733584480718/Put/seqid=0 2024-12-07T15:14:41,687 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3cb6bfad51268ddac87316028a55f638#B#compaction#198 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:14:41,687 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/B/cc06d1fdb66442e1a8b5232e296c7a8d is 50, key is test_row_0/B:col10/1733584480718/Put/seqid=0 2024-12-07T15:14:41,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/A/9e4ce26322904875b1d98fc73ec4a212 is 50, key is test_row_0/A:col10/1733584480732/Put/seqid=0 2024-12-07T15:14:41,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742067_1243 (size=12404) 2024-12-07T15:14:41,725 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/A/5d14fc84f2c440c2abe9f767050c001d as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/5d14fc84f2c440c2abe9f767050c001d 2024-12-07T15:14:41,734 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3cb6bfad51268ddac87316028a55f638/A of 3cb6bfad51268ddac87316028a55f638 into 5d14fc84f2c440c2abe9f767050c001d(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:14:41,734 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3cb6bfad51268ddac87316028a55f638: 2024-12-07T15:14:41,734 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., storeName=3cb6bfad51268ddac87316028a55f638/A, priority=13, startTime=1733584481641; duration=0sec 2024-12-07T15:14:41,734 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T15:14:41,734 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3cb6bfad51268ddac87316028a55f638:A 2024-12-07T15:14:41,734 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:14:41,738 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36303 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:14:41,738 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1541): 3cb6bfad51268ddac87316028a55f638/C is initiating minor compaction (all files) 2024-12-07T15:14:41,738 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3cb6bfad51268ddac87316028a55f638/C in TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:41,738 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/cb9b93545cfe4ce38cce242439564f52, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/f67e7ab9b95d430dbce9903449cb4c47, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/684491ae329b4ad79ea7f6b08d670f80] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp, totalSize=35.5 K 2024-12-07T15:14:41,739 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb9b93545cfe4ce38cce242439564f52, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733584478886 2024-12-07T15:14:41,739 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting f67e7ab9b95d430dbce9903449cb4c47, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733584479026 2024-12-07T15:14:41,739 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 684491ae329b4ad79ea7f6b08d670f80, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1733584480710 2024-12-07T15:14:41,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742068_1244 (size=12404) 2024-12-07T15:14:41,748 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/B/cc06d1fdb66442e1a8b5232e296c7a8d as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/cc06d1fdb66442e1a8b5232e296c7a8d 2024-12-07T15:14:41,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742069_1245 (size=12301) 2024-12-07T15:14:41,752 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/A/9e4ce26322904875b1d98fc73ec4a212 2024-12-07T15:14:41,753 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3cb6bfad51268ddac87316028a55f638#C#compaction#200 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:14:41,754 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/C/c44710a364514d968ffe3579f50a5fab is 50, key is test_row_0/C:col10/1733584480718/Put/seqid=0 2024-12-07T15:14:41,757 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3cb6bfad51268ddac87316028a55f638/B of 3cb6bfad51268ddac87316028a55f638 into cc06d1fdb66442e1a8b5232e296c7a8d(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:14:41,757 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3cb6bfad51268ddac87316028a55f638: 2024-12-07T15:14:41,757 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., storeName=3cb6bfad51268ddac87316028a55f638/B, priority=13, startTime=1733584481641; duration=0sec 2024-12-07T15:14:41,758 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:14:41,758 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3cb6bfad51268ddac87316028a55f638:B 2024-12-07T15:14:41,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/B/984d44aca1604cc18f0ca6eb2d0fa3c5 is 50, key is test_row_0/B:col10/1733584480732/Put/seqid=0 2024-12-07T15:14:41,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742070_1246 (size=12404) 2024-12-07T15:14:41,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742071_1247 (size=12301) 2024-12-07T15:14:41,814 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/B/984d44aca1604cc18f0ca6eb2d0fa3c5 2024-12-07T15:14:41,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/C/880e42fc933449b6afbd9e0d26facaed is 50, key is test_row_0/C:col10/1733584480732/Put/seqid=0 2024-12-07T15:14:41,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742072_1248 (size=12301) 2024-12-07T15:14:41,845 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/C/880e42fc933449b6afbd9e0d26facaed 2024-12-07T15:14:41,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/A/9e4ce26322904875b1d98fc73ec4a212 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/9e4ce26322904875b1d98fc73ec4a212 2024-12-07T15:14:41,867 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/9e4ce26322904875b1d98fc73ec4a212, entries=150, sequenceid=302, filesize=12.0 K 2024-12-07T15:14:41,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/B/984d44aca1604cc18f0ca6eb2d0fa3c5 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/984d44aca1604cc18f0ca6eb2d0fa3c5 2024-12-07T15:14:41,875 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/984d44aca1604cc18f0ca6eb2d0fa3c5, entries=150, sequenceid=302, filesize=12.0 K 2024-12-07T15:14:41,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on 3cb6bfad51268ddac87316028a55f638 2024-12-07T15:14:41,876 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. as already flushing 2024-12-07T15:14:41,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/C/880e42fc933449b6afbd9e0d26facaed as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/880e42fc933449b6afbd9e0d26facaed 2024-12-07T15:14:41,883 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/880e42fc933449b6afbd9e0d26facaed, entries=150, sequenceid=302, filesize=12.0 K 2024-12-07T15:14:41,886 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=20.13 KB/20610 for 3cb6bfad51268ddac87316028a55f638 in 224ms, sequenceid=302, compaction requested=false 2024-12-07T15:14:41,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for 3cb6bfad51268ddac87316028a55f638: 2024-12-07T15:14:41,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:41,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-12-07T15:14:41,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-12-07T15:14:41,891 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-12-07T15:14:41,891 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 840 msec 2024-12-07T15:14:41,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on 3cb6bfad51268ddac87316028a55f638 2024-12-07T15:14:41,892 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3cb6bfad51268ddac87316028a55f638 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-07T15:14:41,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3cb6bfad51268ddac87316028a55f638, store=A 2024-12-07T15:14:41,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:14:41,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3cb6bfad51268ddac87316028a55f638, store=B 2024-12-07T15:14:41,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:14:41,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3cb6bfad51268ddac87316028a55f638, store=C 2024-12-07T15:14:41,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:14:41,894 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 848 msec 2024-12-07T15:14:41,907 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/A/71368f30be044cd0a11ecf442aa836c3 is 50, key is test_row_0/A:col10/1733584481887/Put/seqid=0 2024-12-07T15:14:41,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742073_1249 (size=14741) 2024-12-07T15:14:41,931 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/A/71368f30be044cd0a11ecf442aa836c3 2024-12-07T15:14:41,940 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733584481922, max timestamp=1733584481936Num uniques 859; 2024-12-07T15:14:41,941 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-07T15:14:41,943 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733584481922, max timestamp=1733584481936Num uniques 859; 2024-12-07T15:14:41,943 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-07T15:14:41,944 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733584481922, max timestamp=1733584481936Num uniques 859; 2024-12-07T15:14:41,944 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-07T15:14:41,944 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/B/3cde1d755e154205bedb46845d8d57ce is 50, key is test_row_0/B:col10/1733584481887/Put/seqid=0 2024-12-07T15:14:41,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742074_1250 (size=12301) 2024-12-07T15:14:41,962 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/B/3cde1d755e154205bedb46845d8d57ce 2024-12-07T15:14:41,982 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/C/33d7e474111f44558b2cc3f9e15c0ff4 is 50, key is test_row_0/C:col10/1733584481887/Put/seqid=0 2024-12-07T15:14:41,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742075_1251 (size=12301) 2024-12-07T15:14:42,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:42,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40170 deadline: 1733584541969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:42,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:42,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:42,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:42,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:42,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40166 deadline: 1733584541975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:42,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:42,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:42,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:42,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:42,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40098 deadline: 1733584542013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:42,022 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:42,022 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:42,022 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:42,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:42,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40182 deadline: 1733584542013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:42,023 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:42,023 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:42,023 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:42,023 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:42,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40150 deadline: 1733584542021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:42,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:42,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:42,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:42,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:42,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:42,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40170 deadline: 1733584542123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:42,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40166 deadline: 1733584542123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:42,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:42,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:42,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:42,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:42,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:42,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:42,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:42,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:42,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40098 deadline: 1733584542133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:42,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40182 deadline: 1733584542134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:42,139 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:42,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:42,139 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:42,139 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:42,139 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:42,139 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:42,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40150 deadline: 1733584542134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:42,139 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:42,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:42,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:42,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:42,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-07T15:14:42,184 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T15:14:42,184 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T15:14:42,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-12-07T15:14:42,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-07T15:14:42,186 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T15:14:42,187 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T15:14:42,187 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T15:14:42,225 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/C/c44710a364514d968ffe3579f50a5fab as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/c44710a364514d968ffe3579f50a5fab 2024-12-07T15:14:42,231 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3cb6bfad51268ddac87316028a55f638/C of 3cb6bfad51268ddac87316028a55f638 into c44710a364514d968ffe3579f50a5fab(size=12.1 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:14:42,231 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3cb6bfad51268ddac87316028a55f638: 2024-12-07T15:14:42,231 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., storeName=3cb6bfad51268ddac87316028a55f638/C, priority=13, startTime=1733584481641; duration=0sec 2024-12-07T15:14:42,231 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:14:42,231 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3cb6bfad51268ddac87316028a55f638:C 2024-12-07T15:14:42,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-07T15:14:42,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:42,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40170 deadline: 1733584542334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:42,339 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-07T15:14:42,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:42,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:42,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:42,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:42,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. as already flushing 2024-12-07T15:14:42,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:42,340 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:42,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:42,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:42,347 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:42,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40166 deadline: 1733584542343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:42,347 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:42,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:42,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:42,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40098 deadline: 1733584542343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:42,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:42,348 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:42,348 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:42,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:42,348 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:42,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40182 deadline: 1733584542343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:42,348 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:42,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:42,348 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:42,348 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:42,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40150 deadline: 1733584542344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:42,348 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:42,348 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:42,348 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:42,393 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/C/33d7e474111f44558b2cc3f9e15c0ff4 2024-12-07T15:14:42,400 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/A/71368f30be044cd0a11ecf442aa836c3 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/71368f30be044cd0a11ecf442aa836c3 2024-12-07T15:14:42,406 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/71368f30be044cd0a11ecf442aa836c3, entries=200, sequenceid=315, filesize=14.4 K 2024-12-07T15:14:42,407 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/B/3cde1d755e154205bedb46845d8d57ce as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/3cde1d755e154205bedb46845d8d57ce 2024-12-07T15:14:42,412 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/3cde1d755e154205bedb46845d8d57ce, entries=150, sequenceid=315, filesize=12.0 K 2024-12-07T15:14:42,413 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/C/33d7e474111f44558b2cc3f9e15c0ff4 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/33d7e474111f44558b2cc3f9e15c0ff4 2024-12-07T15:14:42,417 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/33d7e474111f44558b2cc3f9e15c0ff4, entries=150, sequenceid=315, filesize=12.0 K 2024-12-07T15:14:42,418 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=129.71 KB/132820 for 3cb6bfad51268ddac87316028a55f638 in 526ms, sequenceid=315, compaction requested=true 2024-12-07T15:14:42,418 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3cb6bfad51268ddac87316028a55f638: 2024-12-07T15:14:42,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3cb6bfad51268ddac87316028a55f638:A, priority=-2147483648, current under compaction store size is 1 2024-12-07T15:14:42,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:14:42,418 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:14:42,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3cb6bfad51268ddac87316028a55f638:B, priority=-2147483648, current under compaction store size is 2 2024-12-07T15:14:42,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:14:42,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3cb6bfad51268ddac87316028a55f638:C, priority=-2147483648, current under compaction store size is 3 2024-12-07T15:14:42,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T15:14:42,418 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:14:42,420 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:14:42,420 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37006 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:14:42,420 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1541): 3cb6bfad51268ddac87316028a55f638/B is initiating minor compaction (all files) 2024-12-07T15:14:42,420 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1541): 3cb6bfad51268ddac87316028a55f638/A is initiating minor compaction (all files) 2024-12-07T15:14:42,420 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3cb6bfad51268ddac87316028a55f638/B in TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:42,420 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3cb6bfad51268ddac87316028a55f638/A in TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:42,420 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/cc06d1fdb66442e1a8b5232e296c7a8d, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/984d44aca1604cc18f0ca6eb2d0fa3c5, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/3cde1d755e154205bedb46845d8d57ce] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp, totalSize=36.1 K 2024-12-07T15:14:42,420 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/5d14fc84f2c440c2abe9f767050c001d, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/9e4ce26322904875b1d98fc73ec4a212, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/71368f30be044cd0a11ecf442aa836c3] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp, totalSize=38.5 K 2024-12-07T15:14:42,422 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting cc06d1fdb66442e1a8b5232e296c7a8d, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1733584480710 2024-12-07T15:14:42,422 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d14fc84f2c440c2abe9f767050c001d, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1733584480710 2024-12-07T15:14:42,422 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 984d44aca1604cc18f0ca6eb2d0fa3c5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1733584480726 2024-12-07T15:14:42,423 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e4ce26322904875b1d98fc73ec4a212, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1733584480726 2024-12-07T15:14:42,423 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 3cde1d755e154205bedb46845d8d57ce, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1733584481885 2024-12-07T15:14:42,423 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 71368f30be044cd0a11ecf442aa836c3, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1733584481885 2024-12-07T15:14:42,433 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3cb6bfad51268ddac87316028a55f638#B#compaction#206 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:14:42,434 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/B/51c092a9bf3448b6a3a09018e329481e is 50, key is test_row_0/B:col10/1733584481887/Put/seqid=0 2024-12-07T15:14:42,438 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3cb6bfad51268ddac87316028a55f638#A#compaction#207 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:14:42,439 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/A/74679813d7844c048189112e297e283a is 50, key is test_row_0/A:col10/1733584481887/Put/seqid=0 2024-12-07T15:14:42,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742076_1252 (size=12507) 2024-12-07T15:14:42,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742077_1253 (size=12507) 2024-12-07T15:14:42,478 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/A/74679813d7844c048189112e297e283a as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/74679813d7844c048189112e297e283a 2024-12-07T15:14:42,488 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3cb6bfad51268ddac87316028a55f638/A of 3cb6bfad51268ddac87316028a55f638 into 74679813d7844c048189112e297e283a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:14:42,488 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3cb6bfad51268ddac87316028a55f638: 2024-12-07T15:14:42,488 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., storeName=3cb6bfad51268ddac87316028a55f638/A, priority=13, startTime=1733584482418; duration=0sec 2024-12-07T15:14:42,488 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T15:14:42,488 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3cb6bfad51268ddac87316028a55f638:A 2024-12-07T15:14:42,488 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:14:42,490 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37006 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:14:42,490 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1541): 3cb6bfad51268ddac87316028a55f638/C is initiating minor compaction (all files) 2024-12-07T15:14:42,490 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3cb6bfad51268ddac87316028a55f638/C in TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:42,490 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/c44710a364514d968ffe3579f50a5fab, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/880e42fc933449b6afbd9e0d26facaed, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/33d7e474111f44558b2cc3f9e15c0ff4] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp, totalSize=36.1 K 2024-12-07T15:14:42,491 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting c44710a364514d968ffe3579f50a5fab, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1733584480710 2024-12-07T15:14:42,492 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 880e42fc933449b6afbd9e0d26facaed, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1733584480726 2024-12-07T15:14:42,492 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 33d7e474111f44558b2cc3f9e15c0ff4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1733584481885 2024-12-07T15:14:42,493 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-07T15:14:42,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:42,494 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing 3cb6bfad51268ddac87316028a55f638 3/3 column families, dataSize=129.71 KB heapSize=370.14 KB 2024-12-07T15:14:42,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3cb6bfad51268ddac87316028a55f638, store=A 2024-12-07T15:14:42,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T15:14:42,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3cb6bfad51268ddac87316028a55f638, store=B 2024-12-07T15:14:42,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T15:14:42,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3cb6bfad51268ddac87316028a55f638, store=C 2024-12-07T15:14:42,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T15:14:42,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-07T15:14:42,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/A/0c8ea77ed2de4a3a980c0f9289645bb3 is 50, key is test_row_0/A:col10/1733584481979/Put/seqid=0 2024-12-07T15:14:42,524 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3cb6bfad51268ddac87316028a55f638#C#compaction#209 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:14:42,525 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/C/e1861a17d0ff4f16aa0217d06b6509b1 is 50, key is test_row_0/C:col10/1733584481887/Put/seqid=0 2024-12-07T15:14:42,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742079_1255 (size=12507) 2024-12-07T15:14:42,556 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/C/e1861a17d0ff4f16aa0217d06b6509b1 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/e1861a17d0ff4f16aa0217d06b6509b1 2024-12-07T15:14:42,563 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3cb6bfad51268ddac87316028a55f638/C of 3cb6bfad51268ddac87316028a55f638 into e1861a17d0ff4f16aa0217d06b6509b1(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:14:42,563 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3cb6bfad51268ddac87316028a55f638: 2024-12-07T15:14:42,563 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., storeName=3cb6bfad51268ddac87316028a55f638/C, priority=13, startTime=1733584482418; duration=0sec 2024-12-07T15:14:42,564 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:14:42,564 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3cb6bfad51268ddac87316028a55f638:C 2024-12-07T15:14:42,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742078_1254 (size=12301) 2024-12-07T15:14:42,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on 3cb6bfad51268ddac87316028a55f638 2024-12-07T15:14:42,647 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. as already flushing 2024-12-07T15:14:42,667 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:42,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40166 deadline: 1733584542659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:42,667 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:42,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:42,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40182 deadline: 1733584542660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:42,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:42,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:42,668 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:42,668 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:42,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:42,668 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:42,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40098 deadline: 1733584542660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:42,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:42,668 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:42,668 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:42,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40150 deadline: 1733584542660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:42,668 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:42,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:42,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:42,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:42,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:42,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40170 deadline: 1733584542667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:42,674 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:42,674 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:42,674 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:42,778 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:42,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40166 deadline: 1733584542773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:42,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:42,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:42,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:42,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:42,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40098 deadline: 1733584542773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:42,779 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:42,779 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:42,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:42,779 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:42,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40182 deadline: 1733584542773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:42,780 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:42,780 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:42,780 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:42,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:42,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40150 deadline: 1733584542773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:42,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:42,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:42,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:42,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:42,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40170 deadline: 1733584542783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:42,787 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:42,787 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:42,787 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:42,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-07T15:14:42,857 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/B/51c092a9bf3448b6a3a09018e329481e as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/51c092a9bf3448b6a3a09018e329481e 2024-12-07T15:14:42,863 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3cb6bfad51268ddac87316028a55f638/B of 3cb6bfad51268ddac87316028a55f638 into 51c092a9bf3448b6a3a09018e329481e(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:14:42,863 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3cb6bfad51268ddac87316028a55f638: 2024-12-07T15:14:42,863 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., storeName=3cb6bfad51268ddac87316028a55f638/B, priority=13, startTime=1733584482418; duration=0sec 2024-12-07T15:14:42,863 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:14:42,863 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3cb6bfad51268ddac87316028a55f638:B 2024-12-07T15:14:42,961 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/A/0c8ea77ed2de4a3a980c0f9289645bb3 2024-12-07T15:14:42,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/B/5f463b7d437c4bbdbec72beb4fc69d4d is 50, key is test_row_0/B:col10/1733584481979/Put/seqid=0 2024-12-07T15:14:42,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:42,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40166 deadline: 1733584542984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:42,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:42,987 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:42,988 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:42,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40098 deadline: 1733584542984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:42,988 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:42,988 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:42,988 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:42,988 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:42,988 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:42,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40150 deadline: 1733584542984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:42,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:42,988 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:42,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:42,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:42,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40182 deadline: 1733584542984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:42,989 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:42,989 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:42,989 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:42,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:43,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40170 deadline: 1733584542993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:43,000 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:43,000 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:43,000 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:43,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742080_1256 (size=12301) 2024-12-07T15:14:43,296 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:43,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40098 deadline: 1733584543293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:43,297 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:43,297 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:43,297 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:43,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:43,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40150 deadline: 1733584543293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:43,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:43,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40166 deadline: 1733584543293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:43,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:43,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40182 deadline: 1733584543293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:43,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:43,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:43,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:43,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:43,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:43,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:43,299 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:43,299 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:43,299 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:43,318 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:43,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40170 deadline: 1733584543313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:43,319 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:43,319 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:43,319 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:43,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-07T15:14:43,427 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/B/5f463b7d437c4bbdbec72beb4fc69d4d 2024-12-07T15:14:43,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/C/886de3c24eb3424b83423b683c2103cd is 50, key is test_row_0/C:col10/1733584481979/Put/seqid=0 2024-12-07T15:14:43,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742081_1257 (size=12301) 2024-12-07T15:14:43,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:43,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40150 deadline: 1733584543804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:43,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:43,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:43,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:43,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40098 deadline: 1733584543804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:43,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:43,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:43,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40182 deadline: 1733584543804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:43,808 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:43,808 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:43,808 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:43,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:43,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40166 deadline: 1733584543805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:43,808 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:43,808 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:43,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:43,808 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:43,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:43,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:43,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:43,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40170 deadline: 1733584543833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:43,839 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:43,839 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:43,839 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:43,908 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/C/886de3c24eb3424b83423b683c2103cd 2024-12-07T15:14:43,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/A/0c8ea77ed2de4a3a980c0f9289645bb3 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/0c8ea77ed2de4a3a980c0f9289645bb3 2024-12-07T15:14:43,919 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/0c8ea77ed2de4a3a980c0f9289645bb3, entries=150, sequenceid=353, filesize=12.0 K 2024-12-07T15:14:43,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/B/5f463b7d437c4bbdbec72beb4fc69d4d as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/5f463b7d437c4bbdbec72beb4fc69d4d 2024-12-07T15:14:43,925 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/5f463b7d437c4bbdbec72beb4fc69d4d, entries=150, sequenceid=353, filesize=12.0 K 2024-12-07T15:14:43,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/C/886de3c24eb3424b83423b683c2103cd as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/886de3c24eb3424b83423b683c2103cd 2024-12-07T15:14:43,945 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/886de3c24eb3424b83423b683c2103cd, entries=150, sequenceid=353, filesize=12.0 K 2024-12-07T15:14:43,947 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~129.71 KB/132820, heapSize ~370.09 KB/378976, currentSize=53.67 KB/54960 for 3cb6bfad51268ddac87316028a55f638 in 1452ms, sequenceid=353, compaction requested=false 2024-12-07T15:14:43,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for 3cb6bfad51268ddac87316028a55f638: 2024-12-07T15:14:43,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:43,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-12-07T15:14:43,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-12-07T15:14:43,953 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-12-07T15:14:43,953 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7620 sec 2024-12-07T15:14:43,955 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 1.7690 sec 2024-12-07T15:14:44,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-07T15:14:44,334 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T15:14:44,334 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T15:14:44,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-12-07T15:14:44,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-07T15:14:44,336 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T15:14:44,337 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T15:14:44,337 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T15:14:44,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-07T15:14:44,490 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-07T15:14:44,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:44,490 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 3cb6bfad51268ddac87316028a55f638 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-07T15:14:44,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3cb6bfad51268ddac87316028a55f638, store=A 2024-12-07T15:14:44,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:14:44,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3cb6bfad51268ddac87316028a55f638, store=B 2024-12-07T15:14:44,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:14:44,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3cb6bfad51268ddac87316028a55f638, store=C 2024-12-07T15:14:44,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:14:44,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/A/25764b046b2b44588043c04765c36b27 is 50, key is test_row_0/A:col10/1733584482657/Put/seqid=0 2024-12-07T15:14:44,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742082_1258 (size=12301) 2024-12-07T15:14:44,507 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/A/25764b046b2b44588043c04765c36b27 2024-12-07T15:14:44,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/B/530286df9c77447ba86614bc73e2ce13 is 50, key is test_row_0/B:col10/1733584482657/Put/seqid=0 2024-12-07T15:14:44,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742083_1259 (size=12301) 2024-12-07T15:14:44,545 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/B/530286df9c77447ba86614bc73e2ce13 2024-12-07T15:14:44,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/C/16d29d662d924da3bd3c7db74f5c2c26 is 50, key is test_row_0/C:col10/1733584482657/Put/seqid=0 2024-12-07T15:14:44,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742084_1260 (size=12301) 2024-12-07T15:14:44,571 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/C/16d29d662d924da3bd3c7db74f5c2c26 2024-12-07T15:14:44,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/A/25764b046b2b44588043c04765c36b27 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/25764b046b2b44588043c04765c36b27 2024-12-07T15:14:44,581 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/25764b046b2b44588043c04765c36b27, entries=150, sequenceid=366, filesize=12.0 K 2024-12-07T15:14:44,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/B/530286df9c77447ba86614bc73e2ce13 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/530286df9c77447ba86614bc73e2ce13 2024-12-07T15:14:44,587 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/530286df9c77447ba86614bc73e2ce13, entries=150, sequenceid=366, filesize=12.0 K 2024-12-07T15:14:44,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/C/16d29d662d924da3bd3c7db74f5c2c26 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/16d29d662d924da3bd3c7db74f5c2c26 2024-12-07T15:14:44,599 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/16d29d662d924da3bd3c7db74f5c2c26, entries=150, sequenceid=366, filesize=12.0 K 2024-12-07T15:14:44,600 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=0 B/0 for 3cb6bfad51268ddac87316028a55f638 in 110ms, sequenceid=366, compaction requested=true 2024-12-07T15:14:44,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 3cb6bfad51268ddac87316028a55f638: 2024-12-07T15:14:44,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:44,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-12-07T15:14:44,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-12-07T15:14:44,604 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-12-07T15:14:44,604 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 265 msec 2024-12-07T15:14:44,608 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 271 msec 2024-12-07T15:14:44,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-07T15:14:44,653 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T15:14:44,654 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T15:14:44,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-12-07T15:14:44,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-07T15:14:44,656 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T15:14:44,657 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T15:14:44,657 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T15:14:44,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-07T15:14:44,810 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-07T15:14:44,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:44,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 3cb6bfad51268ddac87316028a55f638: 2024-12-07T15:14:44,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:44,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-12-07T15:14:44,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-12-07T15:14:44,815 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-12-07T15:14:44,815 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 156 msec 2024-12-07T15:14:44,817 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 161 msec 2024-12-07T15:14:44,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on 3cb6bfad51268ddac87316028a55f638 2024-12-07T15:14:44,831 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3cb6bfad51268ddac87316028a55f638 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-07T15:14:44,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3cb6bfad51268ddac87316028a55f638, store=A 2024-12-07T15:14:44,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:14:44,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3cb6bfad51268ddac87316028a55f638, store=B 2024-12-07T15:14:44,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:14:44,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3cb6bfad51268ddac87316028a55f638, store=C 2024-12-07T15:14:44,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:14:44,837 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/A/191ba8c3737840dd9273b7a64febd3a2 is 50, key is test_row_0/A:col10/1733584484824/Put/seqid=0 2024-12-07T15:14:44,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742085_1261 (size=17181) 2024-12-07T15:14:44,897 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733584484873, max timestamp=1733584484893Num uniques 859; 2024-12-07T15:14:44,897 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-07T15:14:44,897 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733584484873, max timestamp=1733584484893Num uniques 859; 2024-12-07T15:14:44,897 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-07T15:14:44,897 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733584484873, max timestamp=1733584484893Num uniques 859; 2024-12-07T15:14:44,897 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-07T15:14:44,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:44,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40170 deadline: 1733584544922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:44,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:44,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:44,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:44,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:44,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40182 deadline: 1733584544926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:44,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:44,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:44,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:44,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:44,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40150 deadline: 1733584544927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:44,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:44,935 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:44,935 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:44,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40098 deadline: 1733584544928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:44,935 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:44,935 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:44,935 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:44,935 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:44,936 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:44,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40166 deadline: 1733584544929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:44,936 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:44,936 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:44,936 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:44,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-07T15:14:44,974 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T15:14:44,974 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T15:14:44,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-12-07T15:14:44,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-07T15:14:44,977 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T15:14:44,978 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T15:14:44,978 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T15:14:45,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:45,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40170 deadline: 1733584545033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:45,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:45,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:45,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:45,049 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:45,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40182 deadline: 1733584545043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:45,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:45,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40150 deadline: 1733584545043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:45,049 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:45,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40166 deadline: 1733584545043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:45,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:45,050 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:45,050 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:45,050 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:45,050 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:45,050 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:45,050 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:45,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40098 deadline: 1733584545044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:45,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:45,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:45,050 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:45,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:45,050 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:45,050 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:45,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-07T15:14:45,132 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-07T15:14:45,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:45,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. as already flushing 2024-12-07T15:14:45,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:45,133 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:45,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:45,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:45,247 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=377 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/A/191ba8c3737840dd9273b7a64febd3a2 2024-12-07T15:14:45,248 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:45,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40170 deadline: 1733584545243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:45,249 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:45,249 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:45,249 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:45,255 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/B/2145644a542145b4ad17d0b921aabab6 is 50, key is test_row_0/B:col10/1733584484824/Put/seqid=0 2024-12-07T15:14:45,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:45,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40182 deadline: 1733584545254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:45,258 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:45,258 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:45,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:45,258 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:45,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40166 deadline: 1733584545254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:45,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:45,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:45,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:45,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742086_1262 (size=12301) 2024-12-07T15:14:45,261 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=377 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/B/2145644a542145b4ad17d0b921aabab6 2024-12-07T15:14:45,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:45,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40150 deadline: 1733584545256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:45,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:45,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:45,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:45,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:45,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40098 deadline: 1733584545256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:45,263 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:45,263 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:45,263 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:45,270 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/C/7cc98ccf0d96421c8cdc0c19c0d9831b is 50, key is test_row_0/C:col10/1733584484824/Put/seqid=0 2024-12-07T15:14:45,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742087_1263 (size=12301) 2024-12-07T15:14:45,285 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-07T15:14:45,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:45,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. as already flushing 2024-12-07T15:14:45,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:45,286 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:45,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:45,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:45,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-07T15:14:45,440 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-07T15:14:45,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:45,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. as already flushing 2024-12-07T15:14:45,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:45,441 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:45,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:45,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:45,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:45,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40170 deadline: 1733584545553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:45,560 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:45,560 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:45,560 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:45,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:45,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40182 deadline: 1733584545564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:45,567 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:45,567 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:45,567 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:45,567 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:45,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40166 deadline: 1733584545564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:45,567 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:45,567 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:45,567 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:45,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:45,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40150 deadline: 1733584545573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:45,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:45,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:45,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:45,580 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:45,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40098 deadline: 1733584545574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:45,580 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:45,580 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:45,580 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:45,594 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-07T15:14:45,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:45,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. as already flushing 2024-12-07T15:14:45,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:45,594 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:45,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:45,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:45,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-07T15:14:45,681 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=377 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/C/7cc98ccf0d96421c8cdc0c19c0d9831b 2024-12-07T15:14:45,686 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/A/191ba8c3737840dd9273b7a64febd3a2 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/191ba8c3737840dd9273b7a64febd3a2 2024-12-07T15:14:45,690 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/191ba8c3737840dd9273b7a64febd3a2, entries=250, sequenceid=377, filesize=16.8 K 2024-12-07T15:14:45,691 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/B/2145644a542145b4ad17d0b921aabab6 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/2145644a542145b4ad17d0b921aabab6 2024-12-07T15:14:45,695 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/2145644a542145b4ad17d0b921aabab6, entries=150, sequenceid=377, filesize=12.0 K 2024-12-07T15:14:45,696 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/C/7cc98ccf0d96421c8cdc0c19c0d9831b as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/7cc98ccf0d96421c8cdc0c19c0d9831b 2024-12-07T15:14:45,699 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/7cc98ccf0d96421c8cdc0c19c0d9831b, entries=150, sequenceid=377, filesize=12.0 K 2024-12-07T15:14:45,700 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 3cb6bfad51268ddac87316028a55f638 in 869ms, sequenceid=377, compaction requested=true 2024-12-07T15:14:45,700 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3cb6bfad51268ddac87316028a55f638: 2024-12-07T15:14:45,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3cb6bfad51268ddac87316028a55f638:A, priority=-2147483648, current under compaction store size is 1 2024-12-07T15:14:45,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:14:45,700 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-07T15:14:45,700 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-07T15:14:45,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3cb6bfad51268ddac87316028a55f638:B, priority=-2147483648, current under compaction store size is 2 2024-12-07T15:14:45,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:14:45,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3cb6bfad51268ddac87316028a55f638:C, priority=-2147483648, current under compaction store size is 3 2024-12-07T15:14:45,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T15:14:45,702 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49410 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-07T15:14:45,702 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1541): 3cb6bfad51268ddac87316028a55f638/B is initiating minor compaction (all files) 2024-12-07T15:14:45,702 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3cb6bfad51268ddac87316028a55f638/B in TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:45,702 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/51c092a9bf3448b6a3a09018e329481e, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/5f463b7d437c4bbdbec72beb4fc69d4d, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/530286df9c77447ba86614bc73e2ce13, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/2145644a542145b4ad17d0b921aabab6] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp, totalSize=48.3 K 2024-12-07T15:14:45,703 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54290 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-07T15:14:45,703 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1541): 3cb6bfad51268ddac87316028a55f638/A is initiating minor compaction (all files) 2024-12-07T15:14:45,703 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3cb6bfad51268ddac87316028a55f638/A in TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:45,703 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/74679813d7844c048189112e297e283a, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/0c8ea77ed2de4a3a980c0f9289645bb3, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/25764b046b2b44588043c04765c36b27, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/191ba8c3737840dd9273b7a64febd3a2] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp, totalSize=53.0 K 2024-12-07T15:14:45,703 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 51c092a9bf3448b6a3a09018e329481e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1733584481885 2024-12-07T15:14:45,703 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 74679813d7844c048189112e297e283a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1733584481885 2024-12-07T15:14:45,704 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f463b7d437c4bbdbec72beb4fc69d4d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1733584481965 2024-12-07T15:14:45,704 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c8ea77ed2de4a3a980c0f9289645bb3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1733584481965 2024-12-07T15:14:45,704 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 530286df9c77447ba86614bc73e2ce13, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1733584482657 2024-12-07T15:14:45,704 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 25764b046b2b44588043c04765c36b27, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1733584482657 2024-12-07T15:14:45,705 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 2145644a542145b4ad17d0b921aabab6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1733584484824 2024-12-07T15:14:45,705 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 191ba8c3737840dd9273b7a64febd3a2, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1733584484824 2024-12-07T15:14:45,717 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3cb6bfad51268ddac87316028a55f638#A#compaction#218 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:14:45,718 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/A/0b59732e60304f4d9d2bcb8cd7f54d87 is 50, key is test_row_0/A:col10/1733584484824/Put/seqid=0 2024-12-07T15:14:45,723 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3cb6bfad51268ddac87316028a55f638#B#compaction#219 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:14:45,724 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/B/69057901c6d345c9821dbd22ea0a360c is 50, key is test_row_0/B:col10/1733584484824/Put/seqid=0 2024-12-07T15:14:45,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742088_1264 (size=12643) 2024-12-07T15:14:45,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742089_1265 (size=12643) 2024-12-07T15:14:45,745 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/A/0b59732e60304f4d9d2bcb8cd7f54d87 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/0b59732e60304f4d9d2bcb8cd7f54d87 2024-12-07T15:14:45,748 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-07T15:14:45,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:45,748 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 3cb6bfad51268ddac87316028a55f638 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-12-07T15:14:45,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3cb6bfad51268ddac87316028a55f638, store=A 2024-12-07T15:14:45,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T15:14:45,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3cb6bfad51268ddac87316028a55f638, store=B 2024-12-07T15:14:45,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T15:14:45,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3cb6bfad51268ddac87316028a55f638, store=C 2024-12-07T15:14:45,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T15:14:45,754 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3cb6bfad51268ddac87316028a55f638/A of 3cb6bfad51268ddac87316028a55f638 into 0b59732e60304f4d9d2bcb8cd7f54d87(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:14:45,754 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3cb6bfad51268ddac87316028a55f638: 2024-12-07T15:14:45,754 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., storeName=3cb6bfad51268ddac87316028a55f638/A, priority=12, startTime=1733584485700; duration=0sec 2024-12-07T15:14:45,755 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T15:14:45,755 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3cb6bfad51268ddac87316028a55f638:A 2024-12-07T15:14:45,755 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-07T15:14:45,756 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49410 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-07T15:14:45,756 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1541): 3cb6bfad51268ddac87316028a55f638/C is initiating minor compaction (all files) 2024-12-07T15:14:45,756 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3cb6bfad51268ddac87316028a55f638/C in TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:45,756 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/e1861a17d0ff4f16aa0217d06b6509b1, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/886de3c24eb3424b83423b683c2103cd, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/16d29d662d924da3bd3c7db74f5c2c26, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/7cc98ccf0d96421c8cdc0c19c0d9831b] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp, totalSize=48.3 K 2024-12-07T15:14:45,757 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting e1861a17d0ff4f16aa0217d06b6509b1, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1733584481885 2024-12-07T15:14:45,757 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 886de3c24eb3424b83423b683c2103cd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1733584481965 2024-12-07T15:14:45,758 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 16d29d662d924da3bd3c7db74f5c2c26, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1733584482657 2024-12-07T15:14:45,759 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7cc98ccf0d96421c8cdc0c19c0d9831b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1733584484824 2024-12-07T15:14:45,764 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/B/69057901c6d345c9821dbd22ea0a360c as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/69057901c6d345c9821dbd22ea0a360c 2024-12-07T15:14:45,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/A/cb77b159166d4ae2b6a1d3b285332ce5 is 50, key is test_row_0/A:col10/1733584484926/Put/seqid=0 2024-12-07T15:14:45,774 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3cb6bfad51268ddac87316028a55f638/B of 3cb6bfad51268ddac87316028a55f638 into 69057901c6d345c9821dbd22ea0a360c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:14:45,774 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3cb6bfad51268ddac87316028a55f638: 2024-12-07T15:14:45,774 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., storeName=3cb6bfad51268ddac87316028a55f638/B, priority=12, startTime=1733584485700; duration=0sec 2024-12-07T15:14:45,774 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:14:45,774 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3cb6bfad51268ddac87316028a55f638:B 2024-12-07T15:14:45,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742090_1266 (size=12301) 2024-12-07T15:14:45,790 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=414 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/A/cb77b159166d4ae2b6a1d3b285332ce5 2024-12-07T15:14:45,799 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3cb6bfad51268ddac87316028a55f638#C#compaction#221 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:14:45,800 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/C/a1ebe28579b24c928bee80431d4b6ebb is 50, key is test_row_0/C:col10/1733584484824/Put/seqid=0 2024-12-07T15:14:45,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/B/85e8db7655ac4e57baf53ad04fe77b1d is 50, key is test_row_0/B:col10/1733584484926/Put/seqid=0 2024-12-07T15:14:45,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742091_1267 (size=12643) 2024-12-07T15:14:45,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742092_1268 (size=12301) 2024-12-07T15:14:46,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on 3cb6bfad51268ddac87316028a55f638 2024-12-07T15:14:46,078 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. as already flushing 2024-12-07T15:14:46,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:46,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40166 deadline: 1733584546087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:46,095 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:46,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:46,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40182 deadline: 1733584546087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:46,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:46,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:46,095 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:46,095 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:46,095 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:46,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40098 deadline: 1733584546088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:46,095 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:46,096 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:46,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:46,096 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:46,096 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:46,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40170 deadline: 1733584546088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:46,096 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:46,096 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:46,096 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:46,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:46,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40150 deadline: 1733584546093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:46,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:46,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:46,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:46,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-07T15:14:46,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:46,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40182 deadline: 1733584546203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:46,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:46,206 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:46,207 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:46,207 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:46,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40170 deadline: 1733584546203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:46,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:46,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40166 deadline: 1733584546204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:46,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:46,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40098 deadline: 1733584546205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:46,210 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:46,210 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:46,210 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:46,212 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:46,213 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:46,213 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:46,213 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:46,213 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:46,213 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:46,225 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=414 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/B/85e8db7655ac4e57baf53ad04fe77b1d 2024-12-07T15:14:46,225 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/C/a1ebe28579b24c928bee80431d4b6ebb as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/a1ebe28579b24c928bee80431d4b6ebb 2024-12-07T15:14:46,232 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3cb6bfad51268ddac87316028a55f638/C of 3cb6bfad51268ddac87316028a55f638 into a1ebe28579b24c928bee80431d4b6ebb(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:14:46,232 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3cb6bfad51268ddac87316028a55f638: 2024-12-07T15:14:46,232 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., storeName=3cb6bfad51268ddac87316028a55f638/C, priority=12, startTime=1733584485700; duration=0sec 2024-12-07T15:14:46,232 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:14:46,232 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3cb6bfad51268ddac87316028a55f638:C 2024-12-07T15:14:46,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/C/9d4fa6ef08804d28bdc7ba361ea71ca0 is 50, key is test_row_0/C:col10/1733584484926/Put/seqid=0 2024-12-07T15:14:46,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742093_1269 (size=12301) 2024-12-07T15:14:46,417 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:46,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40182 deadline: 1733584546413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:46,418 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:46,418 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:46,418 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:46,418 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:46,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40098 deadline: 1733584546413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:46,418 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:46,418 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:46,418 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:46,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:46,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40166 deadline: 1733584546423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:46,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:46,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:46,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40170 deadline: 1733584546423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:46,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:46,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:46,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:46,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:46,428 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:46,653 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=414 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/C/9d4fa6ef08804d28bdc7ba361ea71ca0 2024-12-07T15:14:46,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/A/cb77b159166d4ae2b6a1d3b285332ce5 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/cb77b159166d4ae2b6a1d3b285332ce5 2024-12-07T15:14:46,662 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/cb77b159166d4ae2b6a1d3b285332ce5, entries=150, sequenceid=414, filesize=12.0 K 2024-12-07T15:14:46,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/B/85e8db7655ac4e57baf53ad04fe77b1d as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/85e8db7655ac4e57baf53ad04fe77b1d 2024-12-07T15:14:46,669 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/85e8db7655ac4e57baf53ad04fe77b1d, entries=150, sequenceid=414, filesize=12.0 K 2024-12-07T15:14:46,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/C/9d4fa6ef08804d28bdc7ba361ea71ca0 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/9d4fa6ef08804d28bdc7ba361ea71ca0 2024-12-07T15:14:46,674 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/9d4fa6ef08804d28bdc7ba361ea71ca0, entries=150, sequenceid=414, filesize=12.0 K 2024-12-07T15:14:46,676 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=46.96 KB/48090 for 3cb6bfad51268ddac87316028a55f638 in 927ms, sequenceid=414, compaction requested=false 2024-12-07T15:14:46,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 3cb6bfad51268ddac87316028a55f638: 2024-12-07T15:14:46,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:46,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-12-07T15:14:46,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-12-07T15:14:46,680 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-12-07T15:14:46,680 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6990 sec 2024-12-07T15:14:46,682 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.7060 sec 2024-12-07T15:14:46,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on 3cb6bfad51268ddac87316028a55f638 2024-12-07T15:14:46,726 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3cb6bfad51268ddac87316028a55f638 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-07T15:14:46,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3cb6bfad51268ddac87316028a55f638, store=A 2024-12-07T15:14:46,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:14:46,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3cb6bfad51268ddac87316028a55f638, store=B 2024-12-07T15:14:46,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:14:46,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3cb6bfad51268ddac87316028a55f638, store=C 2024-12-07T15:14:46,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:14:46,743 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/A/8c38c9c808ce43a2901bd52521f9b574 is 50, key is test_row_0/A:col10/1733584486085/Put/seqid=0 2024-12-07T15:14:46,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742094_1270 (size=17181) 2024-12-07T15:14:46,748 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/A/8c38c9c808ce43a2901bd52521f9b574 2024-12-07T15:14:46,757 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/B/e69c02b9c1ce44ee8f5e958b1d53427f is 50, key is test_row_0/B:col10/1733584486085/Put/seqid=0 2024-12-07T15:14:46,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:46,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40182 deadline: 1733584546761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:46,769 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:46,769 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:46,769 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:46,770 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733584486750, max timestamp=1733584486767Num uniques 859; 2024-12-07T15:14:46,770 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-07T15:14:46,770 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733584486750, max timestamp=1733584486767Num uniques 859; 2024-12-07T15:14:46,770 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-07T15:14:46,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742095_1271 (size=12301) 2024-12-07T15:14:46,773 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733584486750, max timestamp=1733584486767Num uniques 859; 2024-12-07T15:14:46,773 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-07T15:14:46,774 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/B/e69c02b9c1ce44ee8f5e958b1d53427f 2024-12-07T15:14:46,784 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/C/5df0575bd309491e8c9d0feed602d0a1 is 50, key is test_row_0/C:col10/1733584486085/Put/seqid=0 2024-12-07T15:14:46,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742096_1272 (size=12301) 2024-12-07T15:14:46,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:46,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40170 deadline: 1733584546811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:46,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:46,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40098 deadline: 1733584546812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:46,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:46,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40166 deadline: 1733584546813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:46,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:46,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:46,820 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:46,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:46,820 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:46,820 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:46,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:46,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:46,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:46,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:46,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40182 deadline: 1733584546873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:46,876 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:46,876 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:46,876 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:46,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:46,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40166 deadline: 1733584546923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:46,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:46,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:46,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:46,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40098 deadline: 1733584546924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:46,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:46,927 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:46,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:46,927 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:46,927 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:46,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40170 deadline: 1733584546924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:46,928 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:46,928 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:46,928 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:47,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:47,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40182 deadline: 1733584547083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:47,085 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:47,085 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:47,086 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:47,117 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:47,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40150 deadline: 1733584547113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:47,118 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to d6797ebe9c52,46009,1733584418401 for 'test_row_2' in 3cb6bfad51268ddac87316028a55f638 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-07T15:14:47,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:47,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:47,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:47,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-07T15:14:47,124 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T15:14:47,124 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T15:14:47,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-12-07T15:14:47,126 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T15:14:47,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-07T15:14:47,127 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T15:14:47,127 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T15:14:47,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:47,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40098 deadline: 1733584547134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:47,138 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:47,138 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:47,138 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:47,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:47,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40170 deadline: 1733584547134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:47,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:47,139 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:47,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40166 deadline: 1733584547134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:47,139 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:47,139 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:47,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:47,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:47,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:47,207 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/C/5df0575bd309491e8c9d0feed602d0a1 2024-12-07T15:14:47,212 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/A/8c38c9c808ce43a2901bd52521f9b574 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/8c38c9c808ce43a2901bd52521f9b574 2024-12-07T15:14:47,217 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/8c38c9c808ce43a2901bd52521f9b574, entries=250, sequenceid=429, filesize=16.8 K 2024-12-07T15:14:47,218 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/B/e69c02b9c1ce44ee8f5e958b1d53427f as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/e69c02b9c1ce44ee8f5e958b1d53427f 2024-12-07T15:14:47,222 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/e69c02b9c1ce44ee8f5e958b1d53427f, entries=150, sequenceid=429, filesize=12.0 K 2024-12-07T15:14:47,223 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/C/5df0575bd309491e8c9d0feed602d0a1 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/5df0575bd309491e8c9d0feed602d0a1 2024-12-07T15:14:47,227 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/5df0575bd309491e8c9d0feed602d0a1, entries=150, sequenceid=429, filesize=12.0 K 2024-12-07T15:14:47,228 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=125.23 KB/128240 for 3cb6bfad51268ddac87316028a55f638 in 502ms, sequenceid=429, compaction requested=true 2024-12-07T15:14:47,228 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3cb6bfad51268ddac87316028a55f638: 2024-12-07T15:14:47,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3cb6bfad51268ddac87316028a55f638:A, priority=-2147483648, current under compaction store size is 1 2024-12-07T15:14:47,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:14:47,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3cb6bfad51268ddac87316028a55f638:B, priority=-2147483648, current under compaction store size is 2 2024-12-07T15:14:47,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:14:47,228 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:14:47,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3cb6bfad51268ddac87316028a55f638:C, priority=-2147483648, current under compaction store size is 3 2024-12-07T15:14:47,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T15:14:47,228 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:14:47,229 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37245 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:14:47,229 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42125 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:14:47,229 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1541): 3cb6bfad51268ddac87316028a55f638/B is initiating minor compaction (all files) 2024-12-07T15:14:47,229 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1541): 3cb6bfad51268ddac87316028a55f638/A is initiating minor compaction (all files) 2024-12-07T15:14:47,229 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3cb6bfad51268ddac87316028a55f638/B in TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:47,229 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3cb6bfad51268ddac87316028a55f638/A in TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:47,229 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/0b59732e60304f4d9d2bcb8cd7f54d87, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/cb77b159166d4ae2b6a1d3b285332ce5, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/8c38c9c808ce43a2901bd52521f9b574] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp, totalSize=41.1 K 2024-12-07T15:14:47,229 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/69057901c6d345c9821dbd22ea0a360c, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/85e8db7655ac4e57baf53ad04fe77b1d, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/e69c02b9c1ce44ee8f5e958b1d53427f] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp, totalSize=36.4 K 2024-12-07T15:14:47,230 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b59732e60304f4d9d2bcb8cd7f54d87, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1733584484824 2024-12-07T15:14:47,230 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 69057901c6d345c9821dbd22ea0a360c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1733584484824 2024-12-07T15:14:47,230 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb77b159166d4ae2b6a1d3b285332ce5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1733584484925 2024-12-07T15:14:47,230 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 85e8db7655ac4e57baf53ad04fe77b1d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1733584484925 2024-12-07T15:14:47,230 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c38c9c808ce43a2901bd52521f9b574, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1733584486085 2024-12-07T15:14:47,231 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting e69c02b9c1ce44ee8f5e958b1d53427f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1733584486085 2024-12-07T15:14:47,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-07T15:14:47,244 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3cb6bfad51268ddac87316028a55f638#B#compaction#227 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:14:47,244 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/B/976c3184f3ec4cd49d94571770fbc46c is 50, key is test_row_0/B:col10/1733584486085/Put/seqid=0 2024-12-07T15:14:47,250 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3cb6bfad51268ddac87316028a55f638#A#compaction#228 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:14:47,251 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/A/70f86713795b4cfd8a4bb3b9539e2249 is 50, key is test_row_0/A:col10/1733584486085/Put/seqid=0 2024-12-07T15:14:47,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742097_1273 (size=12745) 2024-12-07T15:14:47,276 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/B/976c3184f3ec4cd49d94571770fbc46c as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/976c3184f3ec4cd49d94571770fbc46c 2024-12-07T15:14:47,279 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-07T15:14:47,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:47,280 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 3cb6bfad51268ddac87316028a55f638 3/3 column families, dataSize=125.23 KB heapSize=350.61 KB 2024-12-07T15:14:47,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3cb6bfad51268ddac87316028a55f638, store=A 2024-12-07T15:14:47,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T15:14:47,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3cb6bfad51268ddac87316028a55f638, store=B 2024-12-07T15:14:47,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T15:14:47,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3cb6bfad51268ddac87316028a55f638, store=C 2024-12-07T15:14:47,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T15:14:47,286 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3cb6bfad51268ddac87316028a55f638/B of 3cb6bfad51268ddac87316028a55f638 into 976c3184f3ec4cd49d94571770fbc46c(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:14:47,286 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3cb6bfad51268ddac87316028a55f638: 2024-12-07T15:14:47,286 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., storeName=3cb6bfad51268ddac87316028a55f638/B, priority=13, startTime=1733584487228; duration=0sec 2024-12-07T15:14:47,286 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T15:14:47,286 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3cb6bfad51268ddac87316028a55f638:B 2024-12-07T15:14:47,286 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:14:47,288 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37245 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:14:47,288 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1541): 3cb6bfad51268ddac87316028a55f638/C is initiating minor compaction (all files) 2024-12-07T15:14:47,288 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3cb6bfad51268ddac87316028a55f638/C in TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:47,288 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/a1ebe28579b24c928bee80431d4b6ebb, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/9d4fa6ef08804d28bdc7ba361ea71ca0, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/5df0575bd309491e8c9d0feed602d0a1] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp, totalSize=36.4 K 2024-12-07T15:14:47,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742098_1274 (size=12745) 2024-12-07T15:14:47,290 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting a1ebe28579b24c928bee80431d4b6ebb, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1733584484824 2024-12-07T15:14:47,290 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d4fa6ef08804d28bdc7ba361ea71ca0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1733584484925 2024-12-07T15:14:47,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/A/b677a6ebc536470e8c68520786896af6 is 50, key is test_row_0/A:col10/1733584486809/Put/seqid=0 2024-12-07T15:14:47,291 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 5df0575bd309491e8c9d0feed602d0a1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1733584486085 2024-12-07T15:14:47,295 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/A/70f86713795b4cfd8a4bb3b9539e2249 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/70f86713795b4cfd8a4bb3b9539e2249 2024-12-07T15:14:47,312 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3cb6bfad51268ddac87316028a55f638/A of 3cb6bfad51268ddac87316028a55f638 into 70f86713795b4cfd8a4bb3b9539e2249(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:14:47,312 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3cb6bfad51268ddac87316028a55f638: 2024-12-07T15:14:47,312 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., storeName=3cb6bfad51268ddac87316028a55f638/A, priority=13, startTime=1733584487228; duration=0sec 2024-12-07T15:14:47,312 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:14:47,312 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3cb6bfad51268ddac87316028a55f638:A 2024-12-07T15:14:47,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742099_1275 (size=12301) 2024-12-07T15:14:47,315 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=463 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/A/b677a6ebc536470e8c68520786896af6 2024-12-07T15:14:47,319 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3cb6bfad51268ddac87316028a55f638#C#compaction#230 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:14:47,320 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/C/b2034b69543e432487736057fea36458 is 50, key is test_row_0/C:col10/1733584486085/Put/seqid=0 2024-12-07T15:14:47,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/B/3cedb7b17be142b4bc2cf7fff8c2fa2b is 50, key is test_row_0/B:col10/1733584486809/Put/seqid=0 2024-12-07T15:14:47,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742100_1276 (size=12745) 2024-12-07T15:14:47,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742101_1277 (size=12301) 2024-12-07T15:14:47,349 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=463 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/B/3cedb7b17be142b4bc2cf7fff8c2fa2b 2024-12-07T15:14:47,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/C/ba19eda8c8f34a448f3274686c1e7a05 is 50, key is test_row_0/C:col10/1733584486809/Put/seqid=0 2024-12-07T15:14:47,399 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. as already flushing 2024-12-07T15:14:47,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on 3cb6bfad51268ddac87316028a55f638 2024-12-07T15:14:47,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742102_1278 (size=12301) 2024-12-07T15:14:47,402 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=463 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/C/ba19eda8c8f34a448f3274686c1e7a05 2024-12-07T15:14:47,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/A/b677a6ebc536470e8c68520786896af6 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/b677a6ebc536470e8c68520786896af6 2024-12-07T15:14:47,414 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/b677a6ebc536470e8c68520786896af6, entries=150, sequenceid=463, filesize=12.0 K 2024-12-07T15:14:47,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/B/3cedb7b17be142b4bc2cf7fff8c2fa2b as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/3cedb7b17be142b4bc2cf7fff8c2fa2b 2024-12-07T15:14:47,422 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/3cedb7b17be142b4bc2cf7fff8c2fa2b, entries=150, sequenceid=463, filesize=12.0 K 2024-12-07T15:14:47,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/C/ba19eda8c8f34a448f3274686c1e7a05 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/ba19eda8c8f34a448f3274686c1e7a05 2024-12-07T15:14:47,428 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/ba19eda8c8f34a448f3274686c1e7a05, entries=150, sequenceid=463, filesize=12.0 K 2024-12-07T15:14:47,429 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~125.23 KB/128240, heapSize ~350.56 KB/358976, currentSize=40.25 KB/41220 for 3cb6bfad51268ddac87316028a55f638 in 149ms, sequenceid=463, compaction requested=false 2024-12-07T15:14:47,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 3cb6bfad51268ddac87316028a55f638: 2024-12-07T15:14:47,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:47,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-12-07T15:14:47,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-12-07T15:14:47,432 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-12-07T15:14:47,432 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 303 msec 2024-12-07T15:14:47,434 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 309 msec 2024-12-07T15:14:47,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-07T15:14:47,444 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T15:14:47,445 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T15:14:47,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-12-07T15:14:47,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-07T15:14:47,447 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T15:14:47,448 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T15:14:47,448 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T15:14:47,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on 3cb6bfad51268ddac87316028a55f638 2024-12-07T15:14:47,449 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3cb6bfad51268ddac87316028a55f638 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-07T15:14:47,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3cb6bfad51268ddac87316028a55f638, store=A 2024-12-07T15:14:47,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:14:47,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3cb6bfad51268ddac87316028a55f638, store=B 2024-12-07T15:14:47,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:14:47,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3cb6bfad51268ddac87316028a55f638, store=C 2024-12-07T15:14:47,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:14:47,459 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/A/769d23839966476c8aa7c76f44a78e74 is 50, key is test_row_0/A:col10/1733584487426/Put/seqid=0 2024-12-07T15:14:47,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742103_1279 (size=12301) 2024-12-07T15:14:47,499 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:47,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40166 deadline: 1733584547491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:47,501 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733584487480, max timestamp=1733584487496Num uniques 859; 2024-12-07T15:14:47,501 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-07T15:14:47,501 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733584487480, max timestamp=1733584487496Num uniques 859; 2024-12-07T15:14:47,502 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=476 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/A/769d23839966476c8aa7c76f44a78e74 2024-12-07T15:14:47,501 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-07T15:14:47,502 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733584487480, max timestamp=1733584487496Num uniques 859; 2024-12-07T15:14:47,502 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-07T15:14:47,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:47,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:47,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:47,513 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/B/0d37ea2c1bc84e64b6187b437f422ba3 is 50, key is test_row_0/B:col10/1733584487426/Put/seqid=0 2024-12-07T15:14:47,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742104_1280 (size=12301) 2024-12-07T15:14:47,546 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=476 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/B/0d37ea2c1bc84e64b6187b437f422ba3 2024-12-07T15:14:47,547 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:47,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40098 deadline: 1733584547539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:47,547 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:47,547 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:47,547 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:47,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-07T15:14:47,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:47,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40182 deadline: 1733584547545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:47,554 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:47,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40170 deadline: 1733584547547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:47,554 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:47,554 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/C/cbdc5ce731e044638cf91be3c10a7eae is 50, key is test_row_0/C:col10/1733584487426/Put/seqid=0 2024-12-07T15:14:47,555 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:47,555 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:47,555 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:47,555 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:47,555 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:47,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742105_1281 (size=12301) 2024-12-07T15:14:47,602 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-07T15:14:47,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:47,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. as already flushing 2024-12-07T15:14:47,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:47,603 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:47,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:47,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:47,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:47,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40166 deadline: 1733584547613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:47,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:47,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:47,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:47,656 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:47,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40098 deadline: 1733584547653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:47,657 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:47,657 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:47,657 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:47,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:47,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40182 deadline: 1733584547663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:47,669 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:47,669 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:47,669 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:47,669 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:47,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40170 deadline: 1733584547663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:47,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:47,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:47,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:47,751 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/C/b2034b69543e432487736057fea36458 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/b2034b69543e432487736057fea36458 2024-12-07T15:14:47,756 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-07T15:14:47,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:47,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. as already flushing 2024-12-07T15:14:47,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:47,756 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:47,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:47,756 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3cb6bfad51268ddac87316028a55f638/C of 3cb6bfad51268ddac87316028a55f638 into b2034b69543e432487736057fea36458(size=12.4 K), total size for store is 24.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:14:47,757 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3cb6bfad51268ddac87316028a55f638: 2024-12-07T15:14:47,757 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., storeName=3cb6bfad51268ddac87316028a55f638/C, priority=13, startTime=1733584487228; duration=0sec 2024-12-07T15:14:47,757 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:14:47,757 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3cb6bfad51268ddac87316028a55f638:C 2024-12-07T15:14:47,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:47,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-07T15:14:47,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:47,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40166 deadline: 1733584547823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:47,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:47,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:47,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:47,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:47,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40098 deadline: 1733584547863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:47,866 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:47,866 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:47,866 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:47,877 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:47,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40182 deadline: 1733584547873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:47,877 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:47,877 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:47,877 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:47,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40170 deadline: 1733584547874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:47,877 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:47,878 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:47,878 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:47,878 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:47,909 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-07T15:14:47,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:47,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. as already flushing 2024-12-07T15:14:47,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:47,909 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:47,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:47,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:47,966 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=476 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/C/cbdc5ce731e044638cf91be3c10a7eae 2024-12-07T15:14:47,972 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/A/769d23839966476c8aa7c76f44a78e74 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/769d23839966476c8aa7c76f44a78e74 2024-12-07T15:14:47,976 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/769d23839966476c8aa7c76f44a78e74, entries=150, sequenceid=476, filesize=12.0 K 2024-12-07T15:14:47,977 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/B/0d37ea2c1bc84e64b6187b437f422ba3 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/0d37ea2c1bc84e64b6187b437f422ba3 2024-12-07T15:14:47,982 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/0d37ea2c1bc84e64b6187b437f422ba3, entries=150, sequenceid=476, filesize=12.0 K 2024-12-07T15:14:47,982 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/C/cbdc5ce731e044638cf91be3c10a7eae as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/cbdc5ce731e044638cf91be3c10a7eae 2024-12-07T15:14:47,987 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/cbdc5ce731e044638cf91be3c10a7eae, entries=150, sequenceid=476, filesize=12.0 K 2024-12-07T15:14:47,988 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 3cb6bfad51268ddac87316028a55f638 in 539ms, sequenceid=476, compaction requested=true 2024-12-07T15:14:47,988 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3cb6bfad51268ddac87316028a55f638: 2024-12-07T15:14:47,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3cb6bfad51268ddac87316028a55f638:A, priority=-2147483648, current under compaction store size is 1 2024-12-07T15:14:47,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:14:47,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3cb6bfad51268ddac87316028a55f638:B, priority=-2147483648, current under compaction store size is 2 2024-12-07T15:14:47,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:14:47,988 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:14:47,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3cb6bfad51268ddac87316028a55f638:C, priority=-2147483648, current under compaction store size is 3 2024-12-07T15:14:47,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T15:14:47,988 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:14:47,997 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37347 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:14:47,997 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37347 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:14:47,997 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1541): 3cb6bfad51268ddac87316028a55f638/A is initiating minor compaction (all files) 2024-12-07T15:14:47,997 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1541): 3cb6bfad51268ddac87316028a55f638/B is initiating minor compaction (all files) 2024-12-07T15:14:47,997 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3cb6bfad51268ddac87316028a55f638/B in TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:47,997 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3cb6bfad51268ddac87316028a55f638/A in TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:47,997 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/976c3184f3ec4cd49d94571770fbc46c, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/3cedb7b17be142b4bc2cf7fff8c2fa2b, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/0d37ea2c1bc84e64b6187b437f422ba3] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp, totalSize=36.5 K 2024-12-07T15:14:47,997 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/70f86713795b4cfd8a4bb3b9539e2249, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/b677a6ebc536470e8c68520786896af6, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/769d23839966476c8aa7c76f44a78e74] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp, totalSize=36.5 K 2024-12-07T15:14:47,998 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 976c3184f3ec4cd49d94571770fbc46c, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1733584486085 2024-12-07T15:14:47,998 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 70f86713795b4cfd8a4bb3b9539e2249, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1733584486085 2024-12-07T15:14:47,998 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting b677a6ebc536470e8c68520786896af6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=463, earliestPutTs=1733584486778 2024-12-07T15:14:47,998 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 3cedb7b17be142b4bc2cf7fff8c2fa2b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=463, earliestPutTs=1733584486778 2024-12-07T15:14:47,998 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 769d23839966476c8aa7c76f44a78e74, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=476, earliestPutTs=1733584487426 2024-12-07T15:14:47,999 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d37ea2c1bc84e64b6187b437f422ba3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=476, earliestPutTs=1733584487426 2024-12-07T15:14:48,006 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3cb6bfad51268ddac87316028a55f638#B#compaction#236 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:14:48,006 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/B/9235aedbfba14158b9ce28d40428fdd3 is 50, key is test_row_0/B:col10/1733584487426/Put/seqid=0 2024-12-07T15:14:48,006 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3cb6bfad51268ddac87316028a55f638#A#compaction#237 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:14:48,007 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/A/648e626a95c64d6b970fe044fb1926c7 is 50, key is test_row_0/A:col10/1733584487426/Put/seqid=0 2024-12-07T15:14:48,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742106_1282 (size=12847) 2024-12-07T15:14:48,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742107_1283 (size=12847) 2024-12-07T15:14:48,062 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-07T15:14:48,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:48,062 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing 3cb6bfad51268ddac87316028a55f638 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-12-07T15:14:48,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3cb6bfad51268ddac87316028a55f638, store=A 2024-12-07T15:14:48,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T15:14:48,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3cb6bfad51268ddac87316028a55f638, store=B 2024-12-07T15:14:48,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T15:14:48,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3cb6bfad51268ddac87316028a55f638, store=C 2024-12-07T15:14:48,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T15:14:48,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/A/9b04d341ea4141c984361d4a48fc5bb2 is 50, key is test_row_0/A:col10/1733584487545/Put/seqid=0 2024-12-07T15:14:48,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-07T15:14:48,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742108_1284 (size=12301) 2024-12-07T15:14:48,078 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=512 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/A/9b04d341ea4141c984361d4a48fc5bb2 2024-12-07T15:14:48,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/B/b4852d986724445abcf9474be4495cbe is 50, key is test_row_0/B:col10/1733584487545/Put/seqid=0 2024-12-07T15:14:48,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742109_1285 (size=12301) 2024-12-07T15:14:48,090 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=512 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/B/b4852d986724445abcf9474be4495cbe 2024-12-07T15:14:48,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/C/ec7315b4edc947efab7c9c11e7d6e354 is 50, key is test_row_0/C:col10/1733584487545/Put/seqid=0 2024-12-07T15:14:48,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742110_1286 (size=12301) 2024-12-07T15:14:48,103 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=512 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/C/ec7315b4edc947efab7c9c11e7d6e354 2024-12-07T15:14:48,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/A/9b04d341ea4141c984361d4a48fc5bb2 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/9b04d341ea4141c984361d4a48fc5bb2 2024-12-07T15:14:48,112 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/9b04d341ea4141c984361d4a48fc5bb2, entries=150, sequenceid=512, filesize=12.0 K 2024-12-07T15:14:48,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/B/b4852d986724445abcf9474be4495cbe as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/b4852d986724445abcf9474be4495cbe 2024-12-07T15:14:48,117 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/b4852d986724445abcf9474be4495cbe, entries=150, sequenceid=512, filesize=12.0 K 2024-12-07T15:14:48,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/C/ec7315b4edc947efab7c9c11e7d6e354 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/ec7315b4edc947efab7c9c11e7d6e354 2024-12-07T15:14:48,122 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/ec7315b4edc947efab7c9c11e7d6e354, entries=150, sequenceid=512, filesize=12.0 K 2024-12-07T15:14:48,123 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=0 B/0 for 3cb6bfad51268ddac87316028a55f638 in 61ms, sequenceid=512, compaction requested=true 2024-12-07T15:14:48,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for 3cb6bfad51268ddac87316028a55f638: 2024-12-07T15:14:48,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:48,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-12-07T15:14:48,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-12-07T15:14:48,127 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-12-07T15:14:48,127 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 676 msec 2024-12-07T15:14:48,131 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 683 msec 2024-12-07T15:14:48,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on 3cb6bfad51268ddac87316028a55f638 2024-12-07T15:14:48,157 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3cb6bfad51268ddac87316028a55f638 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-07T15:14:48,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3cb6bfad51268ddac87316028a55f638, store=A 2024-12-07T15:14:48,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:14:48,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3cb6bfad51268ddac87316028a55f638, store=B 2024-12-07T15:14:48,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:14:48,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3cb6bfad51268ddac87316028a55f638, store=C 2024-12-07T15:14:48,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:14:48,162 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/A/b942999ede43487f9e5f24035c6f6338 is 50, key is test_row_0/A:col10/1733584488134/Put/seqid=0 2024-12-07T15:14:48,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742111_1287 (size=14741) 2024-12-07T15:14:48,207 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:48,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40166 deadline: 1733584548200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:48,207 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733584488197, max timestamp=1733584488204Num uniques 859; 2024-12-07T15:14:48,207 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-07T15:14:48,207 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:48,207 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:48,207 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:48,208 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733584488197, max timestamp=1733584488204Num uniques 859; 2024-12-07T15:14:48,208 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-07T15:14:48,208 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733584488197, max timestamp=1733584488204Num uniques 859; 2024-12-07T15:14:48,208 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-07T15:14:48,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:48,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40098 deadline: 1733584548274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:48,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:48,282 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:48,282 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:48,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40182 deadline: 1733584548275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:48,282 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:48,282 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:48,282 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:48,282 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:48,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:48,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40170 deadline: 1733584548276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:48,283 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:48,283 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:48,283 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:48,319 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:48,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40166 deadline: 1733584548313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:48,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:48,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:48,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:48,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:48,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40098 deadline: 1733584548393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:48,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:48,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40182 deadline: 1733584548393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:48,397 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:48,397 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:48,397 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:48,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:48,397 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:48,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40170 deadline: 1733584548393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:48,397 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:48,398 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:48,398 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:48,398 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:48,398 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:48,422 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/A/648e626a95c64d6b970fe044fb1926c7 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/648e626a95c64d6b970fe044fb1926c7 2024-12-07T15:14:48,422 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/B/9235aedbfba14158b9ce28d40428fdd3 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/9235aedbfba14158b9ce28d40428fdd3 2024-12-07T15:14:48,434 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3cb6bfad51268ddac87316028a55f638/A of 3cb6bfad51268ddac87316028a55f638 into 648e626a95c64d6b970fe044fb1926c7(size=12.5 K), total size for store is 24.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:14:48,434 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3cb6bfad51268ddac87316028a55f638: 2024-12-07T15:14:48,434 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., storeName=3cb6bfad51268ddac87316028a55f638/A, priority=13, startTime=1733584487988; duration=0sec 2024-12-07T15:14:48,434 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3cb6bfad51268ddac87316028a55f638/B of 3cb6bfad51268ddac87316028a55f638 into 9235aedbfba14158b9ce28d40428fdd3(size=12.5 K), total size for store is 24.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:14:48,434 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T15:14:48,434 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3cb6bfad51268ddac87316028a55f638:A 2024-12-07T15:14:48,434 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-07T15:14:48,439 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3cb6bfad51268ddac87316028a55f638: 2024-12-07T15:14:48,439 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., storeName=3cb6bfad51268ddac87316028a55f638/B, priority=13, startTime=1733584487988; duration=0sec 2024-12-07T15:14:48,439 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:14:48,439 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3cb6bfad51268ddac87316028a55f638:B 2024-12-07T15:14:48,440 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49648 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-07T15:14:48,440 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1541): 3cb6bfad51268ddac87316028a55f638/C is initiating minor compaction (all files) 2024-12-07T15:14:48,440 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3cb6bfad51268ddac87316028a55f638/C in TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:48,440 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/b2034b69543e432487736057fea36458, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/ba19eda8c8f34a448f3274686c1e7a05, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/cbdc5ce731e044638cf91be3c10a7eae, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/ec7315b4edc947efab7c9c11e7d6e354] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp, totalSize=48.5 K 2024-12-07T15:14:48,441 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting b2034b69543e432487736057fea36458, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1733584486085 2024-12-07T15:14:48,441 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba19eda8c8f34a448f3274686c1e7a05, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=463, earliestPutTs=1733584486778 2024-12-07T15:14:48,441 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting cbdc5ce731e044638cf91be3c10a7eae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=476, earliestPutTs=1733584487426 2024-12-07T15:14:48,442 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec7315b4edc947efab7c9c11e7d6e354, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=512, earliestPutTs=1733584487535 2024-12-07T15:14:48,459 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3cb6bfad51268ddac87316028a55f638#C#compaction#242 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:14:48,460 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/C/cd659314350f40aaa16809ac7f20ed31 is 50, key is test_row_0/C:col10/1733584487545/Put/seqid=0 2024-12-07T15:14:48,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742112_1288 (size=12881) 2024-12-07T15:14:48,529 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:48,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40166 deadline: 1733584548524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:48,530 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:48,530 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:48,530 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:48,566 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=522 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/A/b942999ede43487f9e5f24035c6f6338 2024-12-07T15:14:48,575 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/B/721e7e982da946d7a05009a0b72e1a8f is 50, key is test_row_0/B:col10/1733584488134/Put/seqid=0 2024-12-07T15:14:48,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-07T15:14:48,584 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T15:14:48,584 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T15:14:48,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-12-07T15:14:48,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-07T15:14:48,588 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T15:14:48,589 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T15:14:48,589 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T15:14:48,608 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:48,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40098 deadline: 1733584548603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:48,608 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:48,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40182 deadline: 1733584548604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:48,609 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:48,609 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:48,609 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:48,609 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:48,609 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:48,609 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:48,609 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:48,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40170 deadline: 1733584548604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:48,610 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:48,610 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:48,610 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:48,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742113_1289 (size=12301) 2024-12-07T15:14:48,620 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=522 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/B/721e7e982da946d7a05009a0b72e1a8f 2024-12-07T15:14:48,633 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/C/3cd7c3dd943249608c3cfa676f72896c is 50, key is test_row_0/C:col10/1733584488134/Put/seqid=0 2024-12-07T15:14:48,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742114_1290 (size=12301) 2024-12-07T15:14:48,643 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=522 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/C/3cd7c3dd943249608c3cfa676f72896c 2024-12-07T15:14:48,651 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/A/b942999ede43487f9e5f24035c6f6338 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/b942999ede43487f9e5f24035c6f6338 2024-12-07T15:14:48,657 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/b942999ede43487f9e5f24035c6f6338, entries=200, sequenceid=522, filesize=14.4 K 2024-12-07T15:14:48,665 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/B/721e7e982da946d7a05009a0b72e1a8f as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/721e7e982da946d7a05009a0b72e1a8f 2024-12-07T15:14:48,670 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/721e7e982da946d7a05009a0b72e1a8f, entries=150, sequenceid=522, filesize=12.0 K 2024-12-07T15:14:48,674 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/C/3cd7c3dd943249608c3cfa676f72896c as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/3cd7c3dd943249608c3cfa676f72896c 2024-12-07T15:14:48,686 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/3cd7c3dd943249608c3cfa676f72896c, entries=150, sequenceid=522, filesize=12.0 K 2024-12-07T15:14:48,687 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 3cb6bfad51268ddac87316028a55f638 in 530ms, sequenceid=522, compaction requested=true 2024-12-07T15:14:48,687 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3cb6bfad51268ddac87316028a55f638: 2024-12-07T15:14:48,688 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:14:48,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3cb6bfad51268ddac87316028a55f638:A, priority=-2147483648, current under compaction store size is 2 2024-12-07T15:14:48,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:14:48,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3cb6bfad51268ddac87316028a55f638:B, priority=-2147483648, current under compaction store size is 3 2024-12-07T15:14:48,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T15:14:48,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3cb6bfad51268ddac87316028a55f638:C, priority=-2147483648, current under compaction store size is 3 2024-12-07T15:14:48,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-07T15:14:48,689 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39889 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:14:48,689 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1541): 3cb6bfad51268ddac87316028a55f638/A is initiating minor compaction (all files) 2024-12-07T15:14:48,689 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3cb6bfad51268ddac87316028a55f638/A in TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:48,689 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/648e626a95c64d6b970fe044fb1926c7, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/9b04d341ea4141c984361d4a48fc5bb2, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/b942999ede43487f9e5f24035c6f6338] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp, totalSize=39.0 K 2024-12-07T15:14:48,689 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 648e626a95c64d6b970fe044fb1926c7, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=476, earliestPutTs=1733584487426 2024-12-07T15:14:48,690 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b04d341ea4141c984361d4a48fc5bb2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=512, earliestPutTs=1733584487535 2024-12-07T15:14:48,690 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting b942999ede43487f9e5f24035c6f6338, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=522, earliestPutTs=1733584488134 2024-12-07T15:14:48,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-07T15:14:48,709 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3cb6bfad51268ddac87316028a55f638#A#compaction#245 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:14:48,710 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/A/c866b0fb764442aa913c2d804c211724 is 50, key is test_row_0/A:col10/1733584488134/Put/seqid=0 2024-12-07T15:14:48,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742115_1291 (size=12949) 2024-12-07T15:14:48,728 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/A/c866b0fb764442aa913c2d804c211724 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/c866b0fb764442aa913c2d804c211724 2024-12-07T15:14:48,736 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3cb6bfad51268ddac87316028a55f638/A of 3cb6bfad51268ddac87316028a55f638 into c866b0fb764442aa913c2d804c211724(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:14:48,736 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3cb6bfad51268ddac87316028a55f638: 2024-12-07T15:14:48,736 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., storeName=3cb6bfad51268ddac87316028a55f638/A, priority=13, startTime=1733584488687; duration=0sec 2024-12-07T15:14:48,736 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-07T15:14:48,736 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3cb6bfad51268ddac87316028a55f638:A 2024-12-07T15:14:48,736 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 4 compacting, 1 eligible, 16 blocking 2024-12-07T15:14:48,737 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-07T15:14:48,737 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-07T15:14:48,737 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. because compaction request was cancelled 2024-12-07T15:14:48,737 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3cb6bfad51268ddac87316028a55f638:C 2024-12-07T15:14:48,737 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:14:48,738 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:14:48,738 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1541): 3cb6bfad51268ddac87316028a55f638/B is initiating minor compaction (all files) 2024-12-07T15:14:48,738 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3cb6bfad51268ddac87316028a55f638/B in TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:48,738 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/9235aedbfba14158b9ce28d40428fdd3, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/b4852d986724445abcf9474be4495cbe, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/721e7e982da946d7a05009a0b72e1a8f] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp, totalSize=36.6 K 2024-12-07T15:14:48,739 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 9235aedbfba14158b9ce28d40428fdd3, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=476, earliestPutTs=1733584487426 2024-12-07T15:14:48,739 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting b4852d986724445abcf9474be4495cbe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=512, earliestPutTs=1733584487535 2024-12-07T15:14:48,740 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 721e7e982da946d7a05009a0b72e1a8f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=522, earliestPutTs=1733584488134 2024-12-07T15:14:48,748 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-07T15:14:48,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:48,749 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing 3cb6bfad51268ddac87316028a55f638 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-07T15:14:48,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3cb6bfad51268ddac87316028a55f638, store=A 2024-12-07T15:14:48,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T15:14:48,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3cb6bfad51268ddac87316028a55f638, store=B 2024-12-07T15:14:48,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T15:14:48,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3cb6bfad51268ddac87316028a55f638, store=C 2024-12-07T15:14:48,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T15:14:48,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/A/30fc2ee5b5744b5ab9a1c92027fdc6f6 is 50, key is test_row_0/A:col10/1733584488273/Put/seqid=0 2024-12-07T15:14:48,766 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3cb6bfad51268ddac87316028a55f638#B#compaction#247 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:14:48,767 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/B/1690aec66e2a4aea9cc5cdf470f89a15 is 50, key is test_row_0/B:col10/1733584488134/Put/seqid=0 2024-12-07T15:14:48,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742117_1293 (size=12949) 2024-12-07T15:14:48,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742116_1292 (size=12301) 2024-12-07T15:14:48,805 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/B/1690aec66e2a4aea9cc5cdf470f89a15 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/1690aec66e2a4aea9cc5cdf470f89a15 2024-12-07T15:14:48,810 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3cb6bfad51268ddac87316028a55f638/B of 3cb6bfad51268ddac87316028a55f638 into 1690aec66e2a4aea9cc5cdf470f89a15(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:14:48,810 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3cb6bfad51268ddac87316028a55f638: 2024-12-07T15:14:48,810 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., storeName=3cb6bfad51268ddac87316028a55f638/B, priority=13, startTime=1733584488688; duration=0sec 2024-12-07T15:14:48,810 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:14:48,810 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3cb6bfad51268ddac87316028a55f638:B 2024-12-07T15:14:48,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on 3cb6bfad51268ddac87316028a55f638 2024-12-07T15:14:48,846 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. as already flushing 2024-12-07T15:14:48,882 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/C/cd659314350f40aaa16809ac7f20ed31 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/cd659314350f40aaa16809ac7f20ed31 2024-12-07T15:14:48,888 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3cb6bfad51268ddac87316028a55f638/C of 3cb6bfad51268ddac87316028a55f638 into cd659314350f40aaa16809ac7f20ed31(size=12.6 K), total size for store is 24.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:14:48,888 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3cb6bfad51268ddac87316028a55f638: 2024-12-07T15:14:48,888 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:48,888 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., storeName=3cb6bfad51268ddac87316028a55f638/C, priority=12, startTime=1733584487988; duration=0sec 2024-12-07T15:14:48,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40166 deadline: 1733584548882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:48,888 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:14:48,888 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3cb6bfad51268ddac87316028a55f638:C 2024-12-07T15:14:48,891 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:48,891 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:48,891 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:48,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-07T15:14:48,918 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:48,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40098 deadline: 1733584548913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:48,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:48,919 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:48,919 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:48,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40182 deadline: 1733584548913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:48,919 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:48,919 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:48,919 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:48,919 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:48,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:48,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40170 deadline: 1733584548913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:48,920 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:48,920 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:48,920 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:48,999 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:49,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40166 deadline: 1733584548993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:49,001 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:49,001 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:49,001 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:49,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:49,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40150 deadline: 1733584549133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:49,137 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to d6797ebe9c52,46009,1733584418401 for 'test_row_2' in 3cb6bfad51268ddac87316028a55f638 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4210 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-07T15:14:49,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:49,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:49,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:49,194 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=562 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/A/30fc2ee5b5744b5ab9a1c92027fdc6f6 2024-12-07T15:14:49,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/B/87f571fc9fc1404e8171fccb95870845 is 50, key is test_row_0/B:col10/1733584488273/Put/seqid=0 2024-12-07T15:14:49,208 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:49,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40166 deadline: 1733584549203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:49,209 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:49,209 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:49,209 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:49,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-07T15:14:49,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742118_1294 (size=12301) 2024-12-07T15:14:49,233 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=562 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/B/87f571fc9fc1404e8171fccb95870845 2024-12-07T15:14:49,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/C/aa5e37572c5b41dd8e2d108bf877ba93 is 50, key is test_row_0/C:col10/1733584488273/Put/seqid=0 2024-12-07T15:14:49,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742119_1295 (size=12301) 2024-12-07T15:14:49,283 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=562 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/C/aa5e37572c5b41dd8e2d108bf877ba93 2024-12-07T15:14:49,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/A/30fc2ee5b5744b5ab9a1c92027fdc6f6 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/30fc2ee5b5744b5ab9a1c92027fdc6f6 2024-12-07T15:14:49,292 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/30fc2ee5b5744b5ab9a1c92027fdc6f6, entries=150, sequenceid=562, filesize=12.0 K 2024-12-07T15:14:49,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/B/87f571fc9fc1404e8171fccb95870845 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/87f571fc9fc1404e8171fccb95870845 2024-12-07T15:14:49,297 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/87f571fc9fc1404e8171fccb95870845, entries=150, sequenceid=562, filesize=12.0 K 2024-12-07T15:14:49,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/C/aa5e37572c5b41dd8e2d108bf877ba93 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/aa5e37572c5b41dd8e2d108bf877ba93 2024-12-07T15:14:49,303 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/aa5e37572c5b41dd8e2d108bf877ba93, entries=150, sequenceid=562, filesize=12.0 K 2024-12-07T15:14:49,304 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 3cb6bfad51268ddac87316028a55f638 in 556ms, sequenceid=562, compaction requested=true 2024-12-07T15:14:49,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for 3cb6bfad51268ddac87316028a55f638: 2024-12-07T15:14:49,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:49,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-12-07T15:14:49,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-12-07T15:14:49,308 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-12-07T15:14:49,308 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 716 msec 2024-12-07T15:14:49,311 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 725 msec 2024-12-07T15:14:49,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on 3cb6bfad51268ddac87316028a55f638 2024-12-07T15:14:49,427 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3cb6bfad51268ddac87316028a55f638 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-07T15:14:49,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3cb6bfad51268ddac87316028a55f638, store=A 2024-12-07T15:14:49,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:14:49,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3cb6bfad51268ddac87316028a55f638, store=B 2024-12-07T15:14:49,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:14:49,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3cb6bfad51268ddac87316028a55f638, store=C 2024-12-07T15:14:49,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:14:49,434 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/A/8f152eff200b4ad4995940d9f1fc0360 is 50, key is test_row_0/A:col10/1733584488880/Put/seqid=0 2024-12-07T15:14:49,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742120_1296 (size=17181) 2024-12-07T15:14:49,458 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=576 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/A/8f152eff200b4ad4995940d9f1fc0360 2024-12-07T15:14:49,466 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/B/08a487c1b672422ab4fc878f67a3bfd5 is 50, key is test_row_0/B:col10/1733584488880/Put/seqid=0 2024-12-07T15:14:49,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742121_1297 (size=12301) 2024-12-07T15:14:49,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:49,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40182 deadline: 1733584549488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:49,494 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:49,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40098 deadline: 1733584549488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:49,494 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:49,494 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:49,494 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:49,494 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:49,494 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:49,494 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:49,498 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:49,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40170 deadline: 1733584549494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:49,499 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:49,499 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:49,499 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:49,517 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:49,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40166 deadline: 1733584549513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:49,517 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:49,517 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:49,517 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:49,604 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:49,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40182 deadline: 1733584549604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:49,605 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:49,605 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:49,605 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:49,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40098 deadline: 1733584549604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:49,605 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:49,605 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:49,605 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:49,605 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:49,605 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:49,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40170 deadline: 1733584549604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:49,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:49,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:49,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:49,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-07T15:14:49,724 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T15:14:49,725 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T15:14:49,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-12-07T15:14:49,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-07T15:14:49,727 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T15:14:49,729 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T15:14:49,729 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T15:14:49,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:49,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40182 deadline: 1733584549813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:49,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:49,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40098 deadline: 1733584549813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:49,815 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:49,815 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:49,815 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:49,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:49,815 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:49,815 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:49,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40170 deadline: 1733584549813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:49,815 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:49,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:49,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:49,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:49,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-07T15:14:49,870 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=576 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/B/08a487c1b672422ab4fc878f67a3bfd5 2024-12-07T15:14:49,878 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/C/fb156a65081a445e9794ff3d5231ea82 is 50, key is test_row_0/C:col10/1733584488880/Put/seqid=0 2024-12-07T15:14:49,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742122_1298 (size=12301) 2024-12-07T15:14:49,882 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=576 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/C/fb156a65081a445e9794ff3d5231ea82 2024-12-07T15:14:49,883 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-07T15:14:49,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:49,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. as already flushing 2024-12-07T15:14:49,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:49,883 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:49,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:49,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:49,888 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/A/8f152eff200b4ad4995940d9f1fc0360 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/8f152eff200b4ad4995940d9f1fc0360 2024-12-07T15:14:49,892 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/8f152eff200b4ad4995940d9f1fc0360, entries=250, sequenceid=576, filesize=16.8 K 2024-12-07T15:14:49,893 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/B/08a487c1b672422ab4fc878f67a3bfd5 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/08a487c1b672422ab4fc878f67a3bfd5 2024-12-07T15:14:49,896 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/08a487c1b672422ab4fc878f67a3bfd5, entries=150, sequenceid=576, filesize=12.0 K 2024-12-07T15:14:49,897 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/C/fb156a65081a445e9794ff3d5231ea82 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/fb156a65081a445e9794ff3d5231ea82 2024-12-07T15:14:49,901 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/fb156a65081a445e9794ff3d5231ea82, entries=150, sequenceid=576, filesize=12.0 K 2024-12-07T15:14:49,902 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 3cb6bfad51268ddac87316028a55f638 in 474ms, sequenceid=576, compaction requested=true 2024-12-07T15:14:49,902 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3cb6bfad51268ddac87316028a55f638: 2024-12-07T15:14:49,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3cb6bfad51268ddac87316028a55f638:A, priority=-2147483648, current under compaction store size is 1 2024-12-07T15:14:49,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:14:49,902 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:14:49,902 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:14:49,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3cb6bfad51268ddac87316028a55f638:B, priority=-2147483648, current under compaction store size is 2 2024-12-07T15:14:49,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:14:49,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3cb6bfad51268ddac87316028a55f638:C, priority=-2147483648, current under compaction store size is 3 2024-12-07T15:14:49,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T15:14:49,903 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:14:49,903 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42431 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:14:49,903 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1541): 3cb6bfad51268ddac87316028a55f638/A is initiating minor compaction (all files) 2024-12-07T15:14:49,903 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1541): 3cb6bfad51268ddac87316028a55f638/B is initiating minor compaction (all files) 2024-12-07T15:14:49,903 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3cb6bfad51268ddac87316028a55f638/A in TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:49,903 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3cb6bfad51268ddac87316028a55f638/B in TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:49,903 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/c866b0fb764442aa913c2d804c211724, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/30fc2ee5b5744b5ab9a1c92027fdc6f6, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/8f152eff200b4ad4995940d9f1fc0360] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp, totalSize=41.4 K 2024-12-07T15:14:49,903 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/1690aec66e2a4aea9cc5cdf470f89a15, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/87f571fc9fc1404e8171fccb95870845, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/08a487c1b672422ab4fc878f67a3bfd5] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp, totalSize=36.7 K 2024-12-07T15:14:49,904 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 1690aec66e2a4aea9cc5cdf470f89a15, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=522, earliestPutTs=1733584488134 2024-12-07T15:14:49,904 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting c866b0fb764442aa913c2d804c211724, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=522, earliestPutTs=1733584488134 2024-12-07T15:14:49,905 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 30fc2ee5b5744b5ab9a1c92027fdc6f6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=562, earliestPutTs=1733584488244 2024-12-07T15:14:49,905 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 87f571fc9fc1404e8171fccb95870845, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=562, earliestPutTs=1733584488244 2024-12-07T15:14:49,905 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8f152eff200b4ad4995940d9f1fc0360, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=576, earliestPutTs=1733584488862 2024-12-07T15:14:49,905 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 08a487c1b672422ab4fc878f67a3bfd5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=576, earliestPutTs=1733584488880 2024-12-07T15:14:49,913 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3cb6bfad51268ddac87316028a55f638#B#compaction#253 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:14:49,914 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/B/f87f547710c9435a9bbdb5f60d736927 is 50, key is test_row_0/B:col10/1733584488880/Put/seqid=0 2024-12-07T15:14:49,914 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3cb6bfad51268ddac87316028a55f638#A#compaction#254 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:14:49,915 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/A/17fde2ae407a47fa96dbc6da7bab1ba6 is 50, key is test_row_0/A:col10/1733584488880/Put/seqid=0 2024-12-07T15:14:49,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742123_1299 (size=13051) 2024-12-07T15:14:49,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742124_1300 (size=13051) 2024-12-07T15:14:49,923 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/A/17fde2ae407a47fa96dbc6da7bab1ba6 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/17fde2ae407a47fa96dbc6da7bab1ba6 2024-12-07T15:14:49,928 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3cb6bfad51268ddac87316028a55f638/A of 3cb6bfad51268ddac87316028a55f638 into 17fde2ae407a47fa96dbc6da7bab1ba6(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:14:49,928 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3cb6bfad51268ddac87316028a55f638: 2024-12-07T15:14:49,928 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., storeName=3cb6bfad51268ddac87316028a55f638/A, priority=13, startTime=1733584489902; duration=0sec 2024-12-07T15:14:49,929 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T15:14:49,929 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3cb6bfad51268ddac87316028a55f638:A 2024-12-07T15:14:49,929 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-07T15:14:49,930 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49784 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-07T15:14:49,930 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1541): 3cb6bfad51268ddac87316028a55f638/C is initiating minor compaction (all files) 2024-12-07T15:14:49,930 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3cb6bfad51268ddac87316028a55f638/C in TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:49,930 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/cd659314350f40aaa16809ac7f20ed31, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/3cd7c3dd943249608c3cfa676f72896c, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/aa5e37572c5b41dd8e2d108bf877ba93, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/fb156a65081a445e9794ff3d5231ea82] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp, totalSize=48.6 K 2024-12-07T15:14:49,930 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd659314350f40aaa16809ac7f20ed31, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=512, earliestPutTs=1733584487535 2024-12-07T15:14:49,931 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3cd7c3dd943249608c3cfa676f72896c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=522, earliestPutTs=1733584488134 2024-12-07T15:14:49,931 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa5e37572c5b41dd8e2d108bf877ba93, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=562, earliestPutTs=1733584488244 2024-12-07T15:14:49,932 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb156a65081a445e9794ff3d5231ea82, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=576, earliestPutTs=1733584488880 2024-12-07T15:14:49,940 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3cb6bfad51268ddac87316028a55f638#C#compaction#255 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:14:49,940 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/C/a2feb325ccaa4099b9212d06bda4ec02 is 50, key is test_row_0/C:col10/1733584488880/Put/seqid=0 2024-12-07T15:14:49,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742125_1301 (size=13017) 2024-12-07T15:14:50,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on 3cb6bfad51268ddac87316028a55f638 2024-12-07T15:14:50,026 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3cb6bfad51268ddac87316028a55f638 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-07T15:14:50,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3cb6bfad51268ddac87316028a55f638, store=A 2024-12-07T15:14:50,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T15:14:50,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3cb6bfad51268ddac87316028a55f638, store=B 2024-12-07T15:14:50,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T15:14:50,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3cb6bfad51268ddac87316028a55f638, store=C 2024-12-07T15:14:50,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T15:14:50,033 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/A/1dedfeb0e4984ce4859d3f659615d720 is 50, key is test_row_0/A:col10/1733584489457/Put/seqid=0 2024-12-07T15:14:50,036 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-07T15:14:50,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:50,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. as already flushing 2024-12-07T15:14:50,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:50,036 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:50,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:50,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:50,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-07T15:14:50,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742126_1302 (size=12301) 2024-12-07T15:14:50,049 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=598 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/A/1dedfeb0e4984ce4859d3f659615d720 2024-12-07T15:14:50,058 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/B/9c2253b642c24f7eaf3f025fbe64750c is 50, key is test_row_0/B:col10/1733584489457/Put/seqid=0 2024-12-07T15:14:50,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742127_1303 (size=12301) 2024-12-07T15:14:50,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:50,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40166 deadline: 1733584550066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:50,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:50,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:50,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:50,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:50,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40182 deadline: 1733584550123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:50,128 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:50,128 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:50,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40098 deadline: 1733584550124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:50,128 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:50,128 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:50,128 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:50,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40170 deadline: 1733584550124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:50,128 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:50,128 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:50,128 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:50,128 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:50,128 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:50,128 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:50,176 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:50,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40166 deadline: 1733584550173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:50,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:50,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:50,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:50,190 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-07T15:14:50,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:50,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. as already flushing 2024-12-07T15:14:50,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:50,194 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:50,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:50,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:50,324 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/B/f87f547710c9435a9bbdb5f60d736927 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/f87f547710c9435a9bbdb5f60d736927 2024-12-07T15:14:50,330 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3cb6bfad51268ddac87316028a55f638/B of 3cb6bfad51268ddac87316028a55f638 into f87f547710c9435a9bbdb5f60d736927(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:14:50,330 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3cb6bfad51268ddac87316028a55f638: 2024-12-07T15:14:50,330 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., storeName=3cb6bfad51268ddac87316028a55f638/B, priority=13, startTime=1733584489902; duration=0sec 2024-12-07T15:14:50,330 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:14:50,330 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3cb6bfad51268ddac87316028a55f638:B 2024-12-07T15:14:50,347 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-07T15:14:50,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:50,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. as already flushing 2024-12-07T15:14:50,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:50,347 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:50,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:50,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:50,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-07T15:14:50,356 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/C/a2feb325ccaa4099b9212d06bda4ec02 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/a2feb325ccaa4099b9212d06bda4ec02 2024-12-07T15:14:50,361 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3cb6bfad51268ddac87316028a55f638/C of 3cb6bfad51268ddac87316028a55f638 into a2feb325ccaa4099b9212d06bda4ec02(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:14:50,361 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3cb6bfad51268ddac87316028a55f638: 2024-12-07T15:14:50,361 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., storeName=3cb6bfad51268ddac87316028a55f638/C, priority=12, startTime=1733584489902; duration=0sec 2024-12-07T15:14:50,362 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:14:50,362 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3cb6bfad51268ddac87316028a55f638:C 2024-12-07T15:14:50,385 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:50,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40166 deadline: 1733584550383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:50,385 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:50,386 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:50,386 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:50,468 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=598 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/B/9c2253b642c24f7eaf3f025fbe64750c 2024-12-07T15:14:50,485 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/C/d928cb287b6940058081f79629b33a07 is 50, key is test_row_0/C:col10/1733584489457/Put/seqid=0 2024-12-07T15:14:50,501 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-07T15:14:50,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:50,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. as already flushing 2024-12-07T15:14:50,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:50,501 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:50,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:50,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:50,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742128_1304 (size=12301) 2024-12-07T15:14:50,509 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=598 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/C/d928cb287b6940058081f79629b33a07 2024-12-07T15:14:50,516 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/A/1dedfeb0e4984ce4859d3f659615d720 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/1dedfeb0e4984ce4859d3f659615d720 2024-12-07T15:14:50,523 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/1dedfeb0e4984ce4859d3f659615d720, entries=150, sequenceid=598, filesize=12.0 K 2024-12-07T15:14:50,524 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/B/9c2253b642c24f7eaf3f025fbe64750c as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/9c2253b642c24f7eaf3f025fbe64750c 2024-12-07T15:14:50,528 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/9c2253b642c24f7eaf3f025fbe64750c, entries=150, sequenceid=598, filesize=12.0 K 2024-12-07T15:14:50,529 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/C/d928cb287b6940058081f79629b33a07 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/d928cb287b6940058081f79629b33a07 2024-12-07T15:14:50,533 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/d928cb287b6940058081f79629b33a07, entries=150, sequenceid=598, filesize=12.0 K 2024-12-07T15:14:50,534 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 3cb6bfad51268ddac87316028a55f638 in 508ms, sequenceid=598, compaction requested=false 2024-12-07T15:14:50,534 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3cb6bfad51268ddac87316028a55f638: 2024-12-07T15:14:50,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on 3cb6bfad51268ddac87316028a55f638 2024-12-07T15:14:50,637 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3cb6bfad51268ddac87316028a55f638 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-07T15:14:50,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3cb6bfad51268ddac87316028a55f638, store=A 2024-12-07T15:14:50,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:14:50,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3cb6bfad51268ddac87316028a55f638, store=B 2024-12-07T15:14:50,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:14:50,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3cb6bfad51268ddac87316028a55f638, store=C 2024-12-07T15:14:50,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:14:50,642 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/A/8b5565f12da448cca237e52dc244be0b is 50, key is test_row_0/A:col10/1733584490635/Put/seqid=0 2024-12-07T15:14:50,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742129_1305 (size=17181) 2024-12-07T15:14:50,656 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-07T15:14:50,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:50,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. as already flushing 2024-12-07T15:14:50,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:50,657 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:50,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:50,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:50,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:50,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40098 deadline: 1733584550671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:50,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:50,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40182 deadline: 1733584550671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:50,677 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:50,677 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:50,677 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:50,677 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:50,677 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:50,677 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:50,681 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:50,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40170 deadline: 1733584550677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:50,681 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:50,681 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:50,681 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:50,696 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:50,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40166 deadline: 1733584550693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:50,697 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:50,697 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:50,697 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:50,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:50,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40098 deadline: 1733584550783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:50,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:50,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40182 deadline: 1733584550783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:50,787 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:50,787 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:50,787 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:50,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:50,787 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:50,787 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:50,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40170 deadline: 1733584550784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:50,787 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:50,787 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:50,787 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:50,787 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:50,809 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-07T15:14:50,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:50,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. as already flushing 2024-12-07T15:14:50,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:50,809 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:50,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:50,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:50,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-07T15:14:50,962 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-07T15:14:50,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:50,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. as already flushing 2024-12-07T15:14:50,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:50,962 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:50,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:50,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:50,997 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:50,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40098 deadline: 1733584550993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:50,998 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:50,998 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:50,998 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:50,998 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:50,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40182 deadline: 1733584550993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:50,998 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:50,998 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:50,998 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:50,998 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:50,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40170 deadline: 1733584550993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:50,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:50,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:50,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:51,047 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=613 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/A/8b5565f12da448cca237e52dc244be0b 2024-12-07T15:14:51,062 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/B/75b005e0c6364fb8900cc6571f8ebb1d is 50, key is test_row_0/B:col10/1733584490635/Put/seqid=0 2024-12-07T15:14:51,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742130_1306 (size=12301) 2024-12-07T15:14:51,115 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-07T15:14:51,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:51,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. as already flushing 2024-12-07T15:14:51,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:51,116 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:51,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:51,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:51,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:51,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40166 deadline: 1733584551203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:51,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:51,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:51,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:51,268 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-07T15:14:51,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:51,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. as already flushing 2024-12-07T15:14:51,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:51,269 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:51,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:51,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:51,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:51,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:51,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40098 deadline: 1733584551303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:51,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40182 deadline: 1733584551303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:51,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:51,307 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:51,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40170 deadline: 1733584551303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:51,307 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:51,307 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:51,307 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:51,307 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:51,307 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:51,307 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:51,307 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:51,307 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:51,421 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-07T15:14:51,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:51,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. as already flushing 2024-12-07T15:14:51,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:51,422 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:51,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:51,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:51,466 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=613 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/B/75b005e0c6364fb8900cc6571f8ebb1d 2024-12-07T15:14:51,475 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/C/56c5bf01f5f0403a8aeab06851623216 is 50, key is test_row_0/C:col10/1733584490635/Put/seqid=0 2024-12-07T15:14:51,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742131_1307 (size=12301) 2024-12-07T15:14:51,575 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-07T15:14:51,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:51,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. as already flushing 2024-12-07T15:14:51,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:51,575 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:51,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:51,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:51,728 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-07T15:14:51,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:51,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. as already flushing 2024-12-07T15:14:51,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:51,728 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:51,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:51,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:51,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:51,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40098 deadline: 1733584551813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:51,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:51,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40182 deadline: 1733584551813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:51,816 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:51,816 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:51,816 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:51,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:51,816 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:51,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40170 deadline: 1733584551813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:51,816 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:51,816 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:51,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:51,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:51,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:51,879 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=613 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/C/56c5bf01f5f0403a8aeab06851623216 2024-12-07T15:14:51,881 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-07T15:14:51,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:51,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. as already flushing 2024-12-07T15:14:51,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:51,881 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:51,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:51,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:51,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-07T15:14:51,884 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/A/8b5565f12da448cca237e52dc244be0b as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/8b5565f12da448cca237e52dc244be0b 2024-12-07T15:14:51,888 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/8b5565f12da448cca237e52dc244be0b, entries=250, sequenceid=613, filesize=16.8 K 2024-12-07T15:14:51,889 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/B/75b005e0c6364fb8900cc6571f8ebb1d as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/75b005e0c6364fb8900cc6571f8ebb1d 2024-12-07T15:14:51,892 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/75b005e0c6364fb8900cc6571f8ebb1d, entries=150, sequenceid=613, filesize=12.0 K 2024-12-07T15:14:51,893 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/C/56c5bf01f5f0403a8aeab06851623216 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/56c5bf01f5f0403a8aeab06851623216 2024-12-07T15:14:51,896 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/56c5bf01f5f0403a8aeab06851623216, entries=150, sequenceid=613, filesize=12.0 K 2024-12-07T15:14:51,897 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 3cb6bfad51268ddac87316028a55f638 in 1261ms, sequenceid=613, compaction requested=true 2024-12-07T15:14:51,897 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3cb6bfad51268ddac87316028a55f638: 2024-12-07T15:14:51,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3cb6bfad51268ddac87316028a55f638:A, priority=-2147483648, current under compaction store size is 1 2024-12-07T15:14:51,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:14:51,897 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:14:51,897 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:14:51,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3cb6bfad51268ddac87316028a55f638:B, priority=-2147483648, current under compaction store size is 2 2024-12-07T15:14:51,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:14:51,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3cb6bfad51268ddac87316028a55f638:C, priority=-2147483648, current under compaction store size is 3 2024-12-07T15:14:51,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T15:14:51,898 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42533 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:14:51,898 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1541): 3cb6bfad51268ddac87316028a55f638/A is initiating minor compaction (all files) 2024-12-07T15:14:51,898 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3cb6bfad51268ddac87316028a55f638/A in TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:51,898 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/17fde2ae407a47fa96dbc6da7bab1ba6, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/1dedfeb0e4984ce4859d3f659615d720, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/8b5565f12da448cca237e52dc244be0b] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp, totalSize=41.5 K 2024-12-07T15:14:51,899 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 17fde2ae407a47fa96dbc6da7bab1ba6, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=576, earliestPutTs=1733584488880 2024-12-07T15:14:51,899 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1dedfeb0e4984ce4859d3f659615d720, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=598, earliestPutTs=1733584489457 2024-12-07T15:14:51,899 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8b5565f12da448cca237e52dc244be0b, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=613, earliestPutTs=1733584490046 2024-12-07T15:14:51,899 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:14:51,900 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1541): 3cb6bfad51268ddac87316028a55f638/B is initiating minor compaction (all files) 2024-12-07T15:14:51,900 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3cb6bfad51268ddac87316028a55f638/B in TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:51,900 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/f87f547710c9435a9bbdb5f60d736927, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/9c2253b642c24f7eaf3f025fbe64750c, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/75b005e0c6364fb8900cc6571f8ebb1d] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp, totalSize=36.8 K 2024-12-07T15:14:51,900 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting f87f547710c9435a9bbdb5f60d736927, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=576, earliestPutTs=1733584488880 2024-12-07T15:14:51,901 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c2253b642c24f7eaf3f025fbe64750c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=598, earliestPutTs=1733584489457 2024-12-07T15:14:51,908 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 75b005e0c6364fb8900cc6571f8ebb1d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=613, earliestPutTs=1733584490053 2024-12-07T15:14:51,914 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3cb6bfad51268ddac87316028a55f638#A#compaction#262 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:14:51,915 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/A/b4a665bee89049de8eb4f65ee164f182 is 50, key is test_row_0/A:col10/1733584490635/Put/seqid=0 2024-12-07T15:14:51,917 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3cb6bfad51268ddac87316028a55f638#B#compaction#263 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:14:51,918 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/B/6f193ef207204ac786d860f531fdc4ca is 50, key is test_row_0/B:col10/1733584490635/Put/seqid=0 2024-12-07T15:14:51,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742132_1308 (size=13153) 2024-12-07T15:14:51,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742133_1309 (size=13153) 2024-12-07T15:14:51,937 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/B/6f193ef207204ac786d860f531fdc4ca as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/6f193ef207204ac786d860f531fdc4ca 2024-12-07T15:14:51,942 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3cb6bfad51268ddac87316028a55f638/B of 3cb6bfad51268ddac87316028a55f638 into 6f193ef207204ac786d860f531fdc4ca(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:14:51,942 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3cb6bfad51268ddac87316028a55f638: 2024-12-07T15:14:51,942 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., storeName=3cb6bfad51268ddac87316028a55f638/B, priority=13, startTime=1733584491897; duration=0sec 2024-12-07T15:14:51,942 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T15:14:51,942 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3cb6bfad51268ddac87316028a55f638:B 2024-12-07T15:14:51,942 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:14:51,944 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:14:51,944 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1541): 3cb6bfad51268ddac87316028a55f638/C is initiating minor compaction (all files) 2024-12-07T15:14:51,944 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3cb6bfad51268ddac87316028a55f638/C in TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:51,944 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/a2feb325ccaa4099b9212d06bda4ec02, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/d928cb287b6940058081f79629b33a07, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/56c5bf01f5f0403a8aeab06851623216] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp, totalSize=36.7 K 2024-12-07T15:14:51,944 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting a2feb325ccaa4099b9212d06bda4ec02, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=576, earliestPutTs=1733584488880 2024-12-07T15:14:51,945 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting d928cb287b6940058081f79629b33a07, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=598, earliestPutTs=1733584489457 2024-12-07T15:14:51,945 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 56c5bf01f5f0403a8aeab06851623216, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=613, earliestPutTs=1733584490053 2024-12-07T15:14:51,952 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3cb6bfad51268ddac87316028a55f638#C#compaction#264 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:14:51,952 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/C/7d2bf6adb31749748f173940f47c6ab0 is 50, key is test_row_0/C:col10/1733584490635/Put/seqid=0 2024-12-07T15:14:51,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742134_1310 (size=13119) 2024-12-07T15:14:51,971 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/C/7d2bf6adb31749748f173940f47c6ab0 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/7d2bf6adb31749748f173940f47c6ab0 2024-12-07T15:14:51,978 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3cb6bfad51268ddac87316028a55f638/C of 3cb6bfad51268ddac87316028a55f638 into 7d2bf6adb31749748f173940f47c6ab0(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:14:51,978 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3cb6bfad51268ddac87316028a55f638: 2024-12-07T15:14:51,978 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., storeName=3cb6bfad51268ddac87316028a55f638/C, priority=13, startTime=1733584491897; duration=0sec 2024-12-07T15:14:51,978 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:14:51,978 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3cb6bfad51268ddac87316028a55f638:C 2024-12-07T15:14:52,033 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-07T15:14:52,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:52,034 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing 3cb6bfad51268ddac87316028a55f638 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-07T15:14:52,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3cb6bfad51268ddac87316028a55f638, store=A 2024-12-07T15:14:52,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:14:52,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3cb6bfad51268ddac87316028a55f638, store=B 2024-12-07T15:14:52,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:14:52,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3cb6bfad51268ddac87316028a55f638, store=C 2024-12-07T15:14:52,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:14:52,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/A/aebbe70aa7c842d6ad56790a78c0b508 is 50, key is test_row_0/A:col10/1733584490660/Put/seqid=0 2024-12-07T15:14:52,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742135_1311 (size=12301) 2024-12-07T15:14:52,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on 3cb6bfad51268ddac87316028a55f638 2024-12-07T15:14:52,215 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. as already flushing 2024-12-07T15:14:52,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:52,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40166 deadline: 1733584552255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:52,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:52,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:52,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:52,335 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/A/b4a665bee89049de8eb4f65ee164f182 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/b4a665bee89049de8eb4f65ee164f182 2024-12-07T15:14:52,340 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3cb6bfad51268ddac87316028a55f638/A of 3cb6bfad51268ddac87316028a55f638 into b4a665bee89049de8eb4f65ee164f182(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:14:52,341 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3cb6bfad51268ddac87316028a55f638: 2024-12-07T15:14:52,341 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., storeName=3cb6bfad51268ddac87316028a55f638/A, priority=13, startTime=1733584491897; duration=0sec 2024-12-07T15:14:52,341 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:14:52,341 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3cb6bfad51268ddac87316028a55f638:A 2024-12-07T15:14:52,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:52,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40166 deadline: 1733584552363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:52,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:52,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:52,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:52,457 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=634 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/A/aebbe70aa7c842d6ad56790a78c0b508 2024-12-07T15:14:52,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/B/deef37995e404e629a75549b44c7ca42 is 50, key is test_row_0/B:col10/1733584490660/Put/seqid=0 2024-12-07T15:14:52,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742136_1312 (size=12301) 2024-12-07T15:14:52,470 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=634 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/B/deef37995e404e629a75549b44c7ca42 2024-12-07T15:14:52,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/C/dfbd8ada737f4d70ad806703b13dbde7 is 50, key is test_row_0/C:col10/1733584490660/Put/seqid=0 2024-12-07T15:14:52,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742137_1313 (size=12301) 2024-12-07T15:14:52,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:52,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40166 deadline: 1733584552573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:52,576 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:52,576 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:52,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:52,825 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:52,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40098 deadline: 1733584552823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:52,825 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:52,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40170 deadline: 1733584552824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:52,825 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to d6797ebe9c52,46009,1733584418401 for 'test_row_0' in 3cb6bfad51268ddac87316028a55f638 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-07T15:14:52,826 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:52,826 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:52,826 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:52,826 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to d6797ebe9c52,46009,1733584418401 for 'test_row_1' in 3cb6bfad51268ddac87316028a55f638 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2149 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-07T15:14:52,826 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:52,826 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:52,826 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:52,836 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:52,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40182 deadline: 1733584552833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:52,837 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to d6797ebe9c52,46009,1733584418401 for 'test_row_2' in 3cb6bfad51268ddac87316028a55f638 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-07T15:14:52,837 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:52,837 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:52,837 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:52,882 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=634 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/C/dfbd8ada737f4d70ad806703b13dbde7 2024-12-07T15:14:52,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:52,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40166 deadline: 1733584552883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:52,887 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:52,887 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:52,887 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:52,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/A/aebbe70aa7c842d6ad56790a78c0b508 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/aebbe70aa7c842d6ad56790a78c0b508 2024-12-07T15:14:52,891 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/aebbe70aa7c842d6ad56790a78c0b508, entries=150, sequenceid=634, filesize=12.0 K 2024-12-07T15:14:52,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/B/deef37995e404e629a75549b44c7ca42 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/deef37995e404e629a75549b44c7ca42 2024-12-07T15:14:52,895 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/deef37995e404e629a75549b44c7ca42, entries=150, sequenceid=634, filesize=12.0 K 2024-12-07T15:14:52,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/C/dfbd8ada737f4d70ad806703b13dbde7 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/dfbd8ada737f4d70ad806703b13dbde7 2024-12-07T15:14:52,899 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/dfbd8ada737f4d70ad806703b13dbde7, entries=150, sequenceid=634, filesize=12.0 K 2024-12-07T15:14:52,900 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 3cb6bfad51268ddac87316028a55f638 in 866ms, sequenceid=634, compaction requested=false 2024-12-07T15:14:52,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for 3cb6bfad51268ddac87316028a55f638: 2024-12-07T15:14:52,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:52,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-12-07T15:14:52,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-12-07T15:14:52,903 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-12-07T15:14:52,904 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1730 sec 2024-12-07T15:14:52,906 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 3.1790 sec 2024-12-07T15:14:53,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on 3cb6bfad51268ddac87316028a55f638 2024-12-07T15:14:53,176 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3cb6bfad51268ddac87316028a55f638 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-07T15:14:53,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3cb6bfad51268ddac87316028a55f638, store=A 2024-12-07T15:14:53,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:14:53,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3cb6bfad51268ddac87316028a55f638, store=B 2024-12-07T15:14:53,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:14:53,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3cb6bfad51268ddac87316028a55f638, store=C 2024-12-07T15:14:53,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:14:53,181 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/A/57f2a447abd94af488155ba5b9c51db8 is 50, key is test_row_0/A:col10/1733584492222/Put/seqid=0 2024-12-07T15:14:53,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742138_1314 (size=12301) 2024-12-07T15:14:53,249 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:53,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40150 deadline: 1733584553246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:53,250 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:53,250 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:53,250 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:53,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:53,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40150 deadline: 1733584553353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:53,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:53,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:53,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:53,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:53,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40166 deadline: 1733584553393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:53,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:53,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:53,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:53,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:53,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40150 deadline: 1733584553563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:53,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:53,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:53,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:53,586 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=649 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/A/57f2a447abd94af488155ba5b9c51db8 2024-12-07T15:14:53,594 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/B/3833f9658d2a4893922f57a58b872bbd is 50, key is test_row_0/B:col10/1733584492222/Put/seqid=0 2024-12-07T15:14:53,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742139_1315 (size=12301) 2024-12-07T15:14:53,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:53,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40150 deadline: 1733584553873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:53,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:53,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:53,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:53,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-07T15:14:53,904 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T15:14:53,904 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T15:14:53,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees 2024-12-07T15:14:53,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-07T15:14:53,906 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T15:14:53,907 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T15:14:53,907 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T15:14:54,000 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=649 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/B/3833f9658d2a4893922f57a58b872bbd 2024-12-07T15:14:54,012 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/C/831a55b1048f4581b03da8a915c586ee is 50, key is test_row_0/C:col10/1733584492222/Put/seqid=0 2024-12-07T15:14:54,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-07T15:14:54,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742140_1316 (size=12301) 2024-12-07T15:14:54,060 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-12-07T15:14:54,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:54,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. as already flushing 2024-12-07T15:14:54,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:54,060 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:54,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:54,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:54,216 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-12-07T15:14:54,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:54,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. as already flushing 2024-12-07T15:14:54,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:54,216 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:54,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:54,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:54,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-07T15:14:54,369 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-12-07T15:14:54,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:54,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. as already flushing 2024-12-07T15:14:54,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:54,369 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:54,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:54,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:54,386 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:54,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40150 deadline: 1733584554383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:54,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:54,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:54,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:54,416 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=649 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/C/831a55b1048f4581b03da8a915c586ee 2024-12-07T15:14:54,417 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:54,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40166 deadline: 1733584554413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:54,418 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to d6797ebe9c52,46009,1733584418401 for 'test_row_1' in 3cb6bfad51268ddac87316028a55f638 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-07T15:14:54,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:54,419 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:54,419 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:54,422 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/A/57f2a447abd94af488155ba5b9c51db8 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/57f2a447abd94af488155ba5b9c51db8 2024-12-07T15:14:54,426 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/57f2a447abd94af488155ba5b9c51db8, entries=150, sequenceid=649, filesize=12.0 K 2024-12-07T15:14:54,426 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/B/3833f9658d2a4893922f57a58b872bbd as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/3833f9658d2a4893922f57a58b872bbd 2024-12-07T15:14:54,430 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/3833f9658d2a4893922f57a58b872bbd, entries=150, sequenceid=649, filesize=12.0 K 2024-12-07T15:14:54,430 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/C/831a55b1048f4581b03da8a915c586ee as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/831a55b1048f4581b03da8a915c586ee 2024-12-07T15:14:54,434 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/831a55b1048f4581b03da8a915c586ee, entries=150, sequenceid=649, filesize=12.0 K 2024-12-07T15:14:54,435 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 3cb6bfad51268ddac87316028a55f638 in 1258ms, sequenceid=649, compaction requested=true 2024-12-07T15:14:54,435 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3cb6bfad51268ddac87316028a55f638: 2024-12-07T15:14:54,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3cb6bfad51268ddac87316028a55f638:A, priority=-2147483648, current under compaction store size is 1 2024-12-07T15:14:54,435 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:14:54,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:14:54,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3cb6bfad51268ddac87316028a55f638:B, priority=-2147483648, current under compaction store size is 2 2024-12-07T15:14:54,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:14:54,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3cb6bfad51268ddac87316028a55f638:C, priority=-2147483648, current under compaction store size is 3 2024-12-07T15:14:54,435 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:14:54,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T15:14:54,436 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:14:54,436 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1541): 3cb6bfad51268ddac87316028a55f638/A is initiating minor compaction (all files) 2024-12-07T15:14:54,436 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:14:54,436 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3cb6bfad51268ddac87316028a55f638/A in TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:54,436 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1541): 3cb6bfad51268ddac87316028a55f638/B is initiating minor compaction (all files) 2024-12-07T15:14:54,436 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3cb6bfad51268ddac87316028a55f638/B in TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:54,436 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/b4a665bee89049de8eb4f65ee164f182, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/aebbe70aa7c842d6ad56790a78c0b508, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/57f2a447abd94af488155ba5b9c51db8] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp, totalSize=36.9 K 2024-12-07T15:14:54,436 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/6f193ef207204ac786d860f531fdc4ca, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/deef37995e404e629a75549b44c7ca42, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/3833f9658d2a4893922f57a58b872bbd] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp, totalSize=36.9 K 2024-12-07T15:14:54,436 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting b4a665bee89049de8eb4f65ee164f182, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=613, earliestPutTs=1733584490053 2024-12-07T15:14:54,436 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f193ef207204ac786d860f531fdc4ca, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=613, earliestPutTs=1733584490053 2024-12-07T15:14:54,437 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting aebbe70aa7c842d6ad56790a78c0b508, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=634, earliestPutTs=1733584490660 2024-12-07T15:14:54,437 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting deef37995e404e629a75549b44c7ca42, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=634, earliestPutTs=1733584490660 2024-12-07T15:14:54,437 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 3833f9658d2a4893922f57a58b872bbd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=649, earliestPutTs=1733584492222 2024-12-07T15:14:54,437 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 57f2a447abd94af488155ba5b9c51db8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=649, earliestPutTs=1733584492222 2024-12-07T15:14:54,445 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3cb6bfad51268ddac87316028a55f638#A#compaction#271 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:14:54,445 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3cb6bfad51268ddac87316028a55f638#B#compaction#272 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:14:54,445 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/A/e101b0be2a834dc9ba26773b2f16b212 is 50, key is test_row_0/A:col10/1733584492222/Put/seqid=0 2024-12-07T15:14:54,445 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/B/aa43990985f34eeca848ff3e74071ac2 is 50, key is test_row_0/B:col10/1733584492222/Put/seqid=0 2024-12-07T15:14:54,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742141_1317 (size=13255) 2024-12-07T15:14:54,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742142_1318 (size=13255) 2024-12-07T15:14:54,462 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/B/aa43990985f34eeca848ff3e74071ac2 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/aa43990985f34eeca848ff3e74071ac2 2024-12-07T15:14:54,467 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3cb6bfad51268ddac87316028a55f638/B of 3cb6bfad51268ddac87316028a55f638 into aa43990985f34eeca848ff3e74071ac2(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:14:54,467 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3cb6bfad51268ddac87316028a55f638: 2024-12-07T15:14:54,467 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., storeName=3cb6bfad51268ddac87316028a55f638/B, priority=13, startTime=1733584494435; duration=0sec 2024-12-07T15:14:54,467 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T15:14:54,467 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3cb6bfad51268ddac87316028a55f638:B 2024-12-07T15:14:54,467 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:14:54,468 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:14:54,468 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1541): 3cb6bfad51268ddac87316028a55f638/C is initiating minor compaction (all files) 2024-12-07T15:14:54,468 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3cb6bfad51268ddac87316028a55f638/C in TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:54,468 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/7d2bf6adb31749748f173940f47c6ab0, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/dfbd8ada737f4d70ad806703b13dbde7, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/831a55b1048f4581b03da8a915c586ee] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp, totalSize=36.8 K 2024-12-07T15:14:54,468 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d2bf6adb31749748f173940f47c6ab0, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=613, earliestPutTs=1733584490053 2024-12-07T15:14:54,468 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting dfbd8ada737f4d70ad806703b13dbde7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=634, earliestPutTs=1733584490660 2024-12-07T15:14:54,469 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 831a55b1048f4581b03da8a915c586ee, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=649, earliestPutTs=1733584492222 2024-12-07T15:14:54,477 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3cb6bfad51268ddac87316028a55f638#C#compaction#273 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:14:54,477 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/C/b2959885ce29424691a703366c665b99 is 50, key is test_row_0/C:col10/1733584492222/Put/seqid=0 2024-12-07T15:14:54,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742143_1319 (size=13221) 2024-12-07T15:14:54,521 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-12-07T15:14:54,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:54,522 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2902): Flushing 3cb6bfad51268ddac87316028a55f638 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-07T15:14:54,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3cb6bfad51268ddac87316028a55f638, store=A 2024-12-07T15:14:54,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:14:54,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3cb6bfad51268ddac87316028a55f638, store=B 2024-12-07T15:14:54,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:14:54,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3cb6bfad51268ddac87316028a55f638, store=C 2024-12-07T15:14:54,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:14:54,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/A/6e50cc150dc24620bfe0235cab12e6ac is 50, key is test_row_0/A:col10/1733584493216/Put/seqid=0 2024-12-07T15:14:54,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-07T15:14:54,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742144_1320 (size=12301) 2024-12-07T15:14:54,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on 3cb6bfad51268ddac87316028a55f638 2024-12-07T15:14:54,837 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. as already flushing 2024-12-07T15:14:54,855 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/A/e101b0be2a834dc9ba26773b2f16b212 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/e101b0be2a834dc9ba26773b2f16b212 2024-12-07T15:14:54,860 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3cb6bfad51268ddac87316028a55f638/A of 3cb6bfad51268ddac87316028a55f638 into e101b0be2a834dc9ba26773b2f16b212(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:14:54,860 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3cb6bfad51268ddac87316028a55f638: 2024-12-07T15:14:54,860 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., storeName=3cb6bfad51268ddac87316028a55f638/A, priority=13, startTime=1733584494435; duration=0sec 2024-12-07T15:14:54,860 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:14:54,860 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3cb6bfad51268ddac87316028a55f638:A 2024-12-07T15:14:54,861 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:54,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40098 deadline: 1733584554856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:54,862 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:54,862 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:54,862 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:54,866 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:54,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40170 deadline: 1733584554861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:54,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:54,867 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:54,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40182 deadline: 1733584554861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:54,867 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:54,867 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:54,867 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:54,867 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:54,867 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:54,886 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/C/b2959885ce29424691a703366c665b99 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/b2959885ce29424691a703366c665b99 2024-12-07T15:14:54,896 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3cb6bfad51268ddac87316028a55f638/C of 3cb6bfad51268ddac87316028a55f638 into b2959885ce29424691a703366c665b99(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:14:54,896 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3cb6bfad51268ddac87316028a55f638: 2024-12-07T15:14:54,896 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., storeName=3cb6bfad51268ddac87316028a55f638/C, priority=13, startTime=1733584494435; duration=0sec 2024-12-07T15:14:54,896 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:14:54,896 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3cb6bfad51268ddac87316028a55f638:C 2024-12-07T15:14:54,934 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=668 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/A/6e50cc150dc24620bfe0235cab12e6ac 2024-12-07T15:14:54,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/B/823c2a4923c54692aa583b4b901067ea is 50, key is test_row_0/B:col10/1733584493216/Put/seqid=0 2024-12-07T15:14:54,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742145_1321 (size=12301) 2024-12-07T15:14:54,946 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=668 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/B/823c2a4923c54692aa583b4b901067ea 2024-12-07T15:14:54,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/C/2f2e755bd3974fd39c0d9fe4c8ec4f62 is 50, key is test_row_0/C:col10/1733584493216/Put/seqid=0 2024-12-07T15:14:54,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742146_1322 (size=12301) 2024-12-07T15:14:54,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:54,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40098 deadline: 1733584554973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:54,975 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:54,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:54,975 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:54,976 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:54,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40182 deadline: 1733584554973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:54,976 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:54,976 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:54,976 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:54,976 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:54,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40170 deadline: 1733584554973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:54,976 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:54,976 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:54,976 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:55,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-07T15:14:55,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:55,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40098 deadline: 1733584555183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:55,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:55,185 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:55,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40182 deadline: 1733584555183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:55,186 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:55,186 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:55,186 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:55,186 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:55,186 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:55,186 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:55,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40170 deadline: 1733584555183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:55,186 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:55,186 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:55,186 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:55,356 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=668 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/C/2f2e755bd3974fd39c0d9fe4c8ec4f62 2024-12-07T15:14:55,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/A/6e50cc150dc24620bfe0235cab12e6ac as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/6e50cc150dc24620bfe0235cab12e6ac 2024-12-07T15:14:55,370 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/6e50cc150dc24620bfe0235cab12e6ac, entries=150, sequenceid=668, filesize=12.0 K 2024-12-07T15:14:55,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/B/823c2a4923c54692aa583b4b901067ea as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/823c2a4923c54692aa583b4b901067ea 2024-12-07T15:14:55,387 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/823c2a4923c54692aa583b4b901067ea, entries=150, sequenceid=668, filesize=12.0 K 2024-12-07T15:14:55,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/C/2f2e755bd3974fd39c0d9fe4c8ec4f62 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/2f2e755bd3974fd39c0d9fe4c8ec4f62 2024-12-07T15:14:55,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:55,395 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/2f2e755bd3974fd39c0d9fe4c8ec4f62, entries=150, sequenceid=668, filesize=12.0 K 2024-12-07T15:14:55,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40150 deadline: 1733584555394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:55,396 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to d6797ebe9c52,46009,1733584418401 for 'test_row_1' in 3cb6bfad51268ddac87316028a55f638 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2149 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-07T15:14:55,396 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 3cb6bfad51268ddac87316028a55f638 in 874ms, sequenceid=668, compaction requested=false 2024-12-07T15:14:55,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:55,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:55,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2603): Flush status journal for 3cb6bfad51268ddac87316028a55f638: 2024-12-07T15:14:55,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:55,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:55,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=83 2024-12-07T15:14:55,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4169): Remote procedure done, pid=83 2024-12-07T15:14:55,410 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-12-07T15:14:55,411 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5010 sec 2024-12-07T15:14:55,413 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees in 1.5070 sec 2024-12-07T15:14:55,496 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3cb6bfad51268ddac87316028a55f638 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-07T15:14:55,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3cb6bfad51268ddac87316028a55f638, store=A 2024-12-07T15:14:55,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:14:55,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3cb6bfad51268ddac87316028a55f638, store=B 2024-12-07T15:14:55,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:14:55,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3cb6bfad51268ddac87316028a55f638, store=C 2024-12-07T15:14:55,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:14:55,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on 3cb6bfad51268ddac87316028a55f638 2024-12-07T15:14:55,504 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/A/0db05f8288a7406694d5b3ff06a15491 is 50, key is test_row_0/A:col10/1733584494840/Put/seqid=0 2024-12-07T15:14:55,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742147_1323 (size=14741) 2024-12-07T15:14:55,529 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:55,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40182 deadline: 1733584555522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:55,530 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:55,530 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:55,530 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:55,535 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:55,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40098 deadline: 1733584555529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:55,535 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:55,535 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:55,535 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:55,535 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:55,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40170 deadline: 1733584555532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:55,536 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:55,536 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:55,536 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:55,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:55,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40182 deadline: 1733584555633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:55,638 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:55,638 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:55,638 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:55,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:55,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40098 deadline: 1733584555644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:55,645 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:55,646 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:55,646 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:55,648 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:55,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40170 deadline: 1733584555644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:55,649 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:55,649 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:55,649 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:55,846 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:55,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40182 deadline: 1733584555843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:55,847 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:55,847 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:55,847 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:55,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:55,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40098 deadline: 1733584555853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:55,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:55,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40170 deadline: 1733584555853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:55,856 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:55,856 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:55,856 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:55,856 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:55,856 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:55,856 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:55,917 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=687 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/A/0db05f8288a7406694d5b3ff06a15491 2024-12-07T15:14:55,925 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/B/57592bd7391b444f9e217d3b38b290bd is 50, key is test_row_0/B:col10/1733584494840/Put/seqid=0 2024-12-07T15:14:55,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742148_1324 (size=12301) 2024-12-07T15:14:56,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-07T15:14:56,053 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T15:14:56,054 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T15:14:56,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees 2024-12-07T15:14:56,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-07T15:14:56,055 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T15:14:56,056 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T15:14:56,056 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T15:14:56,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:56,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40182 deadline: 1733584556154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:56,157 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:56,158 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:56,158 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:56,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-07T15:14:56,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:56,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40098 deadline: 1733584556163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:56,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:56,167 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:56,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40170 deadline: 1733584556163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:56,167 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:56,167 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:56,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:56,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:56,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:56,208 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-12-07T15:14:56,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:56,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. as already flushing 2024-12-07T15:14:56,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:56,209 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:56,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:56,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:56,329 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=687 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/B/57592bd7391b444f9e217d3b38b290bd 2024-12-07T15:14:56,339 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/C/d797ec72aa6c43e2a7b3cac5ac984faa is 50, key is test_row_0/C:col10/1733584494840/Put/seqid=0 2024-12-07T15:14:56,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742149_1325 (size=12301) 2024-12-07T15:14:56,362 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-12-07T15:14:56,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:56,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. as already flushing 2024-12-07T15:14:56,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:56,362 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:56,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:56,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:56,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-07T15:14:56,438 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:56,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40166 deadline: 1733584556434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:56,438 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to d6797ebe9c52,46009,1733584418401 for 'test_row_1' in 3cb6bfad51268ddac87316028a55f638 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-07T15:14:56,439 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:56,439 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:56,439 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:56,515 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-12-07T15:14:56,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:56,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. as already flushing 2024-12-07T15:14:56,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:56,516 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:56,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:56,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:56,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:56,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40182 deadline: 1733584556664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:56,668 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-12-07T15:14:56,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:56,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. as already flushing 2024-12-07T15:14:56,669 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:56,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:56,669 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:56,669 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:56,669 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:56,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:56,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:56,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:56,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40098 deadline: 1733584556674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:56,676 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:56,676 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:56,676 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:56,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-07T15:14:56,686 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:56,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40170 deadline: 1733584556683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:56,686 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:56,687 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:56,687 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:56,746 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=687 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/C/d797ec72aa6c43e2a7b3cac5ac984faa 2024-12-07T15:14:56,750 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/A/0db05f8288a7406694d5b3ff06a15491 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/0db05f8288a7406694d5b3ff06a15491 2024-12-07T15:14:56,755 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/0db05f8288a7406694d5b3ff06a15491, entries=200, sequenceid=687, filesize=14.4 K 2024-12-07T15:14:56,757 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/B/57592bd7391b444f9e217d3b38b290bd as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/57592bd7391b444f9e217d3b38b290bd 2024-12-07T15:14:56,761 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/57592bd7391b444f9e217d3b38b290bd, entries=150, sequenceid=687, filesize=12.0 K 2024-12-07T15:14:56,762 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/C/d797ec72aa6c43e2a7b3cac5ac984faa as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/d797ec72aa6c43e2a7b3cac5ac984faa 2024-12-07T15:14:56,767 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/d797ec72aa6c43e2a7b3cac5ac984faa, entries=150, sequenceid=687, filesize=12.0 K 2024-12-07T15:14:56,768 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 3cb6bfad51268ddac87316028a55f638 in 1272ms, sequenceid=687, compaction requested=true 2024-12-07T15:14:56,768 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3cb6bfad51268ddac87316028a55f638: 2024-12-07T15:14:56,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3cb6bfad51268ddac87316028a55f638:A, priority=-2147483648, current under compaction store size is 1 2024-12-07T15:14:56,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:14:56,768 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:14:56,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3cb6bfad51268ddac87316028a55f638:B, priority=-2147483648, current under compaction store size is 2 2024-12-07T15:14:56,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:14:56,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3cb6bfad51268ddac87316028a55f638:C, priority=-2147483648, current under compaction store size is 3 2024-12-07T15:14:56,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T15:14:56,768 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:14:56,773 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:14:56,773 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1541): 3cb6bfad51268ddac87316028a55f638/B is initiating minor compaction (all files) 2024-12-07T15:14:56,773 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3cb6bfad51268ddac87316028a55f638/B in TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:56,773 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/aa43990985f34eeca848ff3e74071ac2, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/823c2a4923c54692aa583b4b901067ea, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/57592bd7391b444f9e217d3b38b290bd] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp, totalSize=37.0 K 2024-12-07T15:14:56,774 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40297 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:14:56,774 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting aa43990985f34eeca848ff3e74071ac2, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=649, earliestPutTs=1733584492222 2024-12-07T15:14:56,774 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1541): 3cb6bfad51268ddac87316028a55f638/A is initiating minor compaction (all files) 2024-12-07T15:14:56,774 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3cb6bfad51268ddac87316028a55f638/A in TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:56,774 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/e101b0be2a834dc9ba26773b2f16b212, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/6e50cc150dc24620bfe0235cab12e6ac, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/0db05f8288a7406694d5b3ff06a15491] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp, totalSize=39.4 K 2024-12-07T15:14:56,774 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 823c2a4923c54692aa583b4b901067ea, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=668, earliestPutTs=1733584493216 2024-12-07T15:14:56,774 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting e101b0be2a834dc9ba26773b2f16b212, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=649, earliestPutTs=1733584492222 2024-12-07T15:14:56,774 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 57592bd7391b444f9e217d3b38b290bd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=687, earliestPutTs=1733584494840 2024-12-07T15:14:56,774 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e50cc150dc24620bfe0235cab12e6ac, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=668, earliestPutTs=1733584493216 2024-12-07T15:14:56,775 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0db05f8288a7406694d5b3ff06a15491, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=687, earliestPutTs=1733584494840 2024-12-07T15:14:56,786 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3cb6bfad51268ddac87316028a55f638#B#compaction#280 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:14:56,786 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/B/d46717666e714973a2536cb9922cbb78 is 50, key is test_row_0/B:col10/1733584494840/Put/seqid=0 2024-12-07T15:14:56,789 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3cb6bfad51268ddac87316028a55f638#A#compaction#281 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:14:56,790 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/A/56734f24de90472d9c32c43d8b41ff23 is 50, key is test_row_0/A:col10/1733584494840/Put/seqid=0 2024-12-07T15:14:56,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742150_1326 (size=13357) 2024-12-07T15:14:56,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742151_1327 (size=13357) 2024-12-07T15:14:56,811 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/B/d46717666e714973a2536cb9922cbb78 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/d46717666e714973a2536cb9922cbb78 2024-12-07T15:14:56,813 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/A/56734f24de90472d9c32c43d8b41ff23 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/56734f24de90472d9c32c43d8b41ff23 2024-12-07T15:14:56,818 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3cb6bfad51268ddac87316028a55f638/A of 3cb6bfad51268ddac87316028a55f638 into 56734f24de90472d9c32c43d8b41ff23(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:14:56,818 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3cb6bfad51268ddac87316028a55f638: 2024-12-07T15:14:56,818 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., storeName=3cb6bfad51268ddac87316028a55f638/A, priority=13, startTime=1733584496768; duration=0sec 2024-12-07T15:14:56,818 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T15:14:56,818 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3cb6bfad51268ddac87316028a55f638:A 2024-12-07T15:14:56,818 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:14:56,819 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:14:56,819 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1541): 3cb6bfad51268ddac87316028a55f638/C is initiating minor compaction (all files) 2024-12-07T15:14:56,819 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3cb6bfad51268ddac87316028a55f638/C in TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:56,819 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/b2959885ce29424691a703366c665b99, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/2f2e755bd3974fd39c0d9fe4c8ec4f62, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/d797ec72aa6c43e2a7b3cac5ac984faa] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp, totalSize=36.9 K 2024-12-07T15:14:56,819 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting b2959885ce29424691a703366c665b99, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=649, earliestPutTs=1733584492222 2024-12-07T15:14:56,820 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f2e755bd3974fd39c0d9fe4c8ec4f62, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=668, earliestPutTs=1733584493216 2024-12-07T15:14:56,820 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting d797ec72aa6c43e2a7b3cac5ac984faa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=687, earliestPutTs=1733584494840 2024-12-07T15:14:56,823 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-12-07T15:14:56,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:56,824 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2902): Flushing 3cb6bfad51268ddac87316028a55f638 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-07T15:14:56,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3cb6bfad51268ddac87316028a55f638, store=A 2024-12-07T15:14:56,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:14:56,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3cb6bfad51268ddac87316028a55f638, store=B 2024-12-07T15:14:56,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:14:56,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3cb6bfad51268ddac87316028a55f638, store=C 2024-12-07T15:14:56,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:14:56,832 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3cb6bfad51268ddac87316028a55f638/B of 3cb6bfad51268ddac87316028a55f638 into d46717666e714973a2536cb9922cbb78(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:14:56,832 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3cb6bfad51268ddac87316028a55f638: 2024-12-07T15:14:56,832 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., storeName=3cb6bfad51268ddac87316028a55f638/B, priority=13, startTime=1733584496768; duration=0sec 2024-12-07T15:14:56,832 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:14:56,832 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3cb6bfad51268ddac87316028a55f638:B 2024-12-07T15:14:56,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/A/77f90b60377a4f6292ff6f1d51739d4f is 50, key is test_row_0/A:col10/1733584495527/Put/seqid=0 2024-12-07T15:14:56,836 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3cb6bfad51268ddac87316028a55f638#C#compaction#283 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:14:56,837 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/C/d43d5ab6de484de6994e7b9aaa273cf4 is 50, key is test_row_0/C:col10/1733584494840/Put/seqid=0 2024-12-07T15:14:56,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742152_1328 (size=12301) 2024-12-07T15:14:56,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742153_1329 (size=13323) 2024-12-07T15:14:56,863 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/C/d43d5ab6de484de6994e7b9aaa273cf4 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/d43d5ab6de484de6994e7b9aaa273cf4 2024-12-07T15:14:56,870 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3cb6bfad51268ddac87316028a55f638/C of 3cb6bfad51268ddac87316028a55f638 into d43d5ab6de484de6994e7b9aaa273cf4(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:14:56,870 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3cb6bfad51268ddac87316028a55f638: 2024-12-07T15:14:56,871 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., storeName=3cb6bfad51268ddac87316028a55f638/C, priority=13, startTime=1733584496768; duration=0sec 2024-12-07T15:14:56,871 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:14:56,871 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3cb6bfad51268ddac87316028a55f638:C 2024-12-07T15:14:57,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-07T15:14:57,241 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=703 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/A/77f90b60377a4f6292ff6f1d51739d4f 2024-12-07T15:14:57,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/B/6e911842dcde4c3f83afdaeedc48acb8 is 50, key is test_row_0/B:col10/1733584495527/Put/seqid=0 2024-12-07T15:14:57,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742154_1330 (size=12301) 2024-12-07T15:14:57,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on 3cb6bfad51268ddac87316028a55f638 2024-12-07T15:14:57,405 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. as already flushing 2024-12-07T15:14:57,475 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:57,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40150 deadline: 1733584557470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:57,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:57,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:57,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:57,585 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:57,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40150 deadline: 1733584557583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:57,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:57,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:57,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:57,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:57,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40182 deadline: 1733584557683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:57,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:57,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40098 deadline: 1733584557683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:57,685 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to d6797ebe9c52,46009,1733584418401 for 'test_row_0' in 3cb6bfad51268ddac87316028a55f638 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-07T15:14:57,686 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:57,686 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:57,686 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:57,686 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to d6797ebe9c52,46009,1733584418401 for 'test_row_1' in 3cb6bfad51268ddac87316028a55f638 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2156 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-07T15:14:57,686 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:57,686 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:57,686 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:57,686 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=703 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/B/6e911842dcde4c3f83afdaeedc48acb8 2024-12-07T15:14:57,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/C/6ad03a78dd4249a083cd1e8162484701 is 50, key is test_row_0/C:col10/1733584495527/Put/seqid=0 2024-12-07T15:14:57,696 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:57,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40170 deadline: 1733584557693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:57,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742155_1331 (size=12301) 2024-12-07T15:14:57,697 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to d6797ebe9c52,46009,1733584418401 for 'test_row_1' in 3cb6bfad51268ddac87316028a55f638 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-07T15:14:57,698 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:57,698 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:57,698 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:57,698 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=703 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/C/6ad03a78dd4249a083cd1e8162484701 2024-12-07T15:14:57,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/A/77f90b60377a4f6292ff6f1d51739d4f as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/77f90b60377a4f6292ff6f1d51739d4f 2024-12-07T15:14:57,706 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/77f90b60377a4f6292ff6f1d51739d4f, entries=150, sequenceid=703, filesize=12.0 K 2024-12-07T15:14:57,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/B/6e911842dcde4c3f83afdaeedc48acb8 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/6e911842dcde4c3f83afdaeedc48acb8 2024-12-07T15:14:57,710 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/6e911842dcde4c3f83afdaeedc48acb8, entries=150, sequenceid=703, filesize=12.0 K 2024-12-07T15:14:57,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/C/6ad03a78dd4249a083cd1e8162484701 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/6ad03a78dd4249a083cd1e8162484701 2024-12-07T15:14:57,715 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/6ad03a78dd4249a083cd1e8162484701, entries=150, sequenceid=703, filesize=12.0 K 2024-12-07T15:14:57,715 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 3cb6bfad51268ddac87316028a55f638 in 891ms, sequenceid=703, compaction requested=false 2024-12-07T15:14:57,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2603): Flush status journal for 3cb6bfad51268ddac87316028a55f638: 2024-12-07T15:14:57,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:57,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=85 2024-12-07T15:14:57,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4169): Remote procedure done, pid=85 2024-12-07T15:14:57,719 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-12-07T15:14:57,719 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6610 sec 2024-12-07T15:14:57,720 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees in 1.6650 sec 2024-12-07T15:14:57,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on 3cb6bfad51268ddac87316028a55f638 2024-12-07T15:14:57,795 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3cb6bfad51268ddac87316028a55f638 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-07T15:14:57,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3cb6bfad51268ddac87316028a55f638, store=A 2024-12-07T15:14:57,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:14:57,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3cb6bfad51268ddac87316028a55f638, store=B 2024-12-07T15:14:57,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:14:57,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3cb6bfad51268ddac87316028a55f638, store=C 2024-12-07T15:14:57,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:14:57,800 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/A/0e8b71e8ce8b4e7b8dd6a0a677b6fdfc is 50, key is test_row_0/A:col10/1733584497468/Put/seqid=0 2024-12-07T15:14:57,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742156_1332 (size=14741) 2024-12-07T15:14:57,847 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:57,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40150 deadline: 1733584557844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:57,848 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:57,848 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:57,848 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:57,955 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:57,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40150 deadline: 1733584557953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:57,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:57,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:57,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:58,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:58,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40150 deadline: 1733584558164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:58,166 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:58,166 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:58,166 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:58,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-07T15:14:58,203 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T15:14:58,203 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T15:14:58,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees 2024-12-07T15:14:58,205 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=723 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/A/0e8b71e8ce8b4e7b8dd6a0a677b6fdfc 2024-12-07T15:14:58,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-12-07T15:14:58,205 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=86, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T15:14:58,206 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=86, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T15:14:58,206 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T15:14:58,214 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/B/d63e87a1d3424925b6e07c179df2eb2f is 50, key is test_row_0/B:col10/1733584497468/Put/seqid=0 2024-12-07T15:14:58,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742157_1333 (size=12301) 2024-12-07T15:14:58,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-12-07T15:14:58,359 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-12-07T15:14:58,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:58,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. as already flushing 2024-12-07T15:14:58,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:58,360 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:58,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:58,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:58,474 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:58,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40150 deadline: 1733584558473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:58,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:58,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:58,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:58,513 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-12-07T15:14:58,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:58,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. as already flushing 2024-12-07T15:14:58,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:58,514 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:58,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:58,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:58,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-12-07T15:14:58,619 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=723 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/B/d63e87a1d3424925b6e07c179df2eb2f 2024-12-07T15:14:58,639 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/C/e76cbbd8243d423c88ac370b7c109737 is 50, key is test_row_0/C:col10/1733584497468/Put/seqid=0 2024-12-07T15:14:58,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742158_1334 (size=12301) 2024-12-07T15:14:58,667 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-12-07T15:14:58,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:58,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. as already flushing 2024-12-07T15:14:58,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:58,668 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:58,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:58,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:58,822 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-12-07T15:14:58,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:58,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. as already flushing 2024-12-07T15:14:58,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:58,822 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:58,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:58,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:58,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-12-07T15:14:58,947 INFO [Thread-1102 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1102. 2024-12-07T15:14:58,947 DEBUG [Thread-1102 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T15:14:58,947 DEBUG [Thread-1102 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:14:58,947 DEBUG [Thread-1102 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:14:58,947 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T15:14:58,948 INFO [Thread-1106 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1106. 2024-12-07T15:14:58,948 DEBUG [Thread-1106 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T15:14:58,948 DEBUG [Thread-1106 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:14:58,948 DEBUG [Thread-1106 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:14:58,948 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T15:14:58,950 INFO [Thread-1104 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1104. 2024-12-07T15:14:58,950 DEBUG [Thread-1104 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T15:14:58,950 DEBUG [Thread-1104 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:14:58,950 DEBUG [Thread-1104 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:14:58,950 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T15:14:58,951 INFO [Thread-1100 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1100. 2024-12-07T15:14:58,951 DEBUG [Thread-1100 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T15:14:58,951 DEBUG [Thread-1100 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:14:58,951 DEBUG [Thread-1100 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:14:58,951 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T15:14:58,954 INFO [Thread-1108 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1108. 2024-12-07T15:14:58,954 DEBUG [Thread-1108 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T15:14:58,954 DEBUG [Thread-1108 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:14:58,954 DEBUG [Thread-1108 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:14:58,954 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T15:14:58,976 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-12-07T15:14:58,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:58,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. as already flushing 2024-12-07T15:14:58,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:58,976 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:58,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:58,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:14:58,984 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:14:58,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40150 deadline: 1733584558984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:14:58,985 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:58,985 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:14:58,985 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:14:59,055 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=723 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/C/e76cbbd8243d423c88ac370b7c109737 2024-12-07T15:14:59,060 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/A/0e8b71e8ce8b4e7b8dd6a0a677b6fdfc as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/0e8b71e8ce8b4e7b8dd6a0a677b6fdfc 2024-12-07T15:14:59,064 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/0e8b71e8ce8b4e7b8dd6a0a677b6fdfc, entries=200, sequenceid=723, filesize=14.4 K 2024-12-07T15:14:59,065 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/B/d63e87a1d3424925b6e07c179df2eb2f as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/d63e87a1d3424925b6e07c179df2eb2f 2024-12-07T15:14:59,069 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/d63e87a1d3424925b6e07c179df2eb2f, entries=150, sequenceid=723, filesize=12.0 K 2024-12-07T15:14:59,070 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/C/e76cbbd8243d423c88ac370b7c109737 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/e76cbbd8243d423c88ac370b7c109737 2024-12-07T15:14:59,073 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/e76cbbd8243d423c88ac370b7c109737, entries=150, sequenceid=723, filesize=12.0 K 2024-12-07T15:14:59,074 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 3cb6bfad51268ddac87316028a55f638 in 1279ms, sequenceid=723, compaction requested=true 2024-12-07T15:14:59,074 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3cb6bfad51268ddac87316028a55f638: 2024-12-07T15:14:59,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3cb6bfad51268ddac87316028a55f638:A, priority=-2147483648, current under compaction store size is 1 2024-12-07T15:14:59,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:14:59,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3cb6bfad51268ddac87316028a55f638:B, priority=-2147483648, current under compaction store size is 2 2024-12-07T15:14:59,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:14:59,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3cb6bfad51268ddac87316028a55f638:C, priority=-2147483648, current under compaction store size is 3 2024-12-07T15:14:59,075 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:14:59,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T15:14:59,075 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:14:59,076 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40399 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:14:59,076 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:14:59,076 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1541): 3cb6bfad51268ddac87316028a55f638/A is initiating minor compaction (all files) 2024-12-07T15:14:59,076 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1541): 3cb6bfad51268ddac87316028a55f638/B is initiating minor compaction (all files) 2024-12-07T15:14:59,076 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3cb6bfad51268ddac87316028a55f638/A in TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:59,076 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3cb6bfad51268ddac87316028a55f638/B in TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:59,076 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/56734f24de90472d9c32c43d8b41ff23, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/77f90b60377a4f6292ff6f1d51739d4f, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/0e8b71e8ce8b4e7b8dd6a0a677b6fdfc] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp, totalSize=39.5 K 2024-12-07T15:14:59,076 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/d46717666e714973a2536cb9922cbb78, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/6e911842dcde4c3f83afdaeedc48acb8, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/d63e87a1d3424925b6e07c179df2eb2f] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp, totalSize=37.1 K 2024-12-07T15:14:59,077 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 56734f24de90472d9c32c43d8b41ff23, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=687, earliestPutTs=1733584494840 2024-12-07T15:14:59,077 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting d46717666e714973a2536cb9922cbb78, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=687, earliestPutTs=1733584494840 2024-12-07T15:14:59,077 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e911842dcde4c3f83afdaeedc48acb8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=703, earliestPutTs=1733584495514 2024-12-07T15:14:59,077 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 77f90b60377a4f6292ff6f1d51739d4f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=703, earliestPutTs=1733584495514 2024-12-07T15:14:59,077 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting d63e87a1d3424925b6e07c179df2eb2f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=723, earliestPutTs=1733584497462 2024-12-07T15:14:59,077 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e8b71e8ce8b4e7b8dd6a0a677b6fdfc, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=723, earliestPutTs=1733584497457 2024-12-07T15:14:59,087 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3cb6bfad51268ddac87316028a55f638#B#compaction#289 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:14:59,087 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/B/0221994b2a4d43168fdce5a12fa3603c is 50, key is test_row_0/B:col10/1733584497468/Put/seqid=0 2024-12-07T15:14:59,088 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3cb6bfad51268ddac87316028a55f638#A#compaction#290 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:14:59,089 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/A/0dda79aa85764d358ca0d7c5546cf553 is 50, key is test_row_0/A:col10/1733584497468/Put/seqid=0 2024-12-07T15:14:59,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742160_1336 (size=13459) 2024-12-07T15:14:59,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742159_1335 (size=13459) 2024-12-07T15:14:59,129 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-12-07T15:14:59,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:59,130 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2902): Flushing 3cb6bfad51268ddac87316028a55f638 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-07T15:14:59,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3cb6bfad51268ddac87316028a55f638, store=A 2024-12-07T15:14:59,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:14:59,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3cb6bfad51268ddac87316028a55f638, store=B 2024-12-07T15:14:59,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:14:59,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3cb6bfad51268ddac87316028a55f638, store=C 2024-12-07T15:14:59,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:14:59,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/A/34d79872ad7c4eac904612021c48f6e7 is 50, key is test_row_0/A:col10/1733584497808/Put/seqid=0 2024-12-07T15:14:59,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742161_1337 (size=12301) 2024-12-07T15:14:59,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-12-07T15:14:59,506 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/A/0dda79aa85764d358ca0d7c5546cf553 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/0dda79aa85764d358ca0d7c5546cf553 2024-12-07T15:14:59,506 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/B/0221994b2a4d43168fdce5a12fa3603c as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/0221994b2a4d43168fdce5a12fa3603c 2024-12-07T15:14:59,512 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3cb6bfad51268ddac87316028a55f638/A of 3cb6bfad51268ddac87316028a55f638 into 0dda79aa85764d358ca0d7c5546cf553(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:14:59,512 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3cb6bfad51268ddac87316028a55f638/B of 3cb6bfad51268ddac87316028a55f638 into 0221994b2a4d43168fdce5a12fa3603c(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:14:59,512 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3cb6bfad51268ddac87316028a55f638: 2024-12-07T15:14:59,512 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3cb6bfad51268ddac87316028a55f638: 2024-12-07T15:14:59,512 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., storeName=3cb6bfad51268ddac87316028a55f638/A, priority=13, startTime=1733584499075; duration=0sec 2024-12-07T15:14:59,512 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., storeName=3cb6bfad51268ddac87316028a55f638/B, priority=13, startTime=1733584499075; duration=0sec 2024-12-07T15:14:59,512 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T15:14:59,512 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3cb6bfad51268ddac87316028a55f638:A 2024-12-07T15:14:59,512 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:14:59,512 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:14:59,512 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3cb6bfad51268ddac87316028a55f638:B 2024-12-07T15:14:59,513 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:14:59,513 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1541): 3cb6bfad51268ddac87316028a55f638/C is initiating minor compaction (all files) 2024-12-07T15:14:59,513 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3cb6bfad51268ddac87316028a55f638/C in TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:59,513 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/d43d5ab6de484de6994e7b9aaa273cf4, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/6ad03a78dd4249a083cd1e8162484701, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/e76cbbd8243d423c88ac370b7c109737] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp, totalSize=37.0 K 2024-12-07T15:14:59,513 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting d43d5ab6de484de6994e7b9aaa273cf4, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=687, earliestPutTs=1733584494840 2024-12-07T15:14:59,514 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ad03a78dd4249a083cd1e8162484701, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=703, earliestPutTs=1733584495514 2024-12-07T15:14:59,514 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting e76cbbd8243d423c88ac370b7c109737, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=723, earliestPutTs=1733584497462 2024-12-07T15:14:59,531 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3cb6bfad51268ddac87316028a55f638#C#compaction#292 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:14:59,531 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/C/ef68d7503ec046599559888b6cbcd2ad is 50, key is test_row_0/C:col10/1733584497468/Put/seqid=0 2024-12-07T15:14:59,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742162_1338 (size=13425) 2024-12-07T15:14:59,538 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=737 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/A/34d79872ad7c4eac904612021c48f6e7 2024-12-07T15:14:59,542 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/C/ef68d7503ec046599559888b6cbcd2ad as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/ef68d7503ec046599559888b6cbcd2ad 2024-12-07T15:14:59,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/B/09caafc2f44743bf88a7fc49d6d63665 is 50, key is test_row_0/B:col10/1733584497808/Put/seqid=0 2024-12-07T15:14:59,551 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3cb6bfad51268ddac87316028a55f638/C of 3cb6bfad51268ddac87316028a55f638 into ef68d7503ec046599559888b6cbcd2ad(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:14:59,551 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3cb6bfad51268ddac87316028a55f638: 2024-12-07T15:14:59,551 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., storeName=3cb6bfad51268ddac87316028a55f638/C, priority=13, startTime=1733584499075; duration=0sec 2024-12-07T15:14:59,551 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:14:59,551 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3cb6bfad51268ddac87316028a55f638:C 2024-12-07T15:14:59,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742163_1339 (size=12301) 2024-12-07T15:14:59,554 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=737 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/B/09caafc2f44743bf88a7fc49d6d63665 2024-12-07T15:14:59,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/C/18ce844a5cf04dd5a148a2817280720e is 50, key is test_row_0/C:col10/1733584497808/Put/seqid=0 2024-12-07T15:14:59,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742164_1340 (size=12301) 2024-12-07T15:14:59,705 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. as already flushing 2024-12-07T15:14:59,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on 3cb6bfad51268ddac87316028a55f638 2024-12-07T15:14:59,705 INFO [Thread-1097 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1097. 2024-12-07T15:14:59,706 INFO [Thread-1095 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1095. 2024-12-07T15:14:59,706 DEBUG [Thread-1097 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T15:14:59,706 DEBUG [Thread-1095 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T15:14:59,706 DEBUG [Thread-1097 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:14:59,706 DEBUG [Thread-1095 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:14:59,706 DEBUG [Thread-1097 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:14:59,706 DEBUG [Thread-1095 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:14:59,706 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T15:14:59,706 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T15:14:59,725 INFO [Thread-1093 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1093. 2024-12-07T15:14:59,725 DEBUG [Thread-1093 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T15:14:59,725 DEBUG [Thread-1093 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:14:59,725 DEBUG [Thread-1093 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:14:59,725 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T15:14:59,976 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=737 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/C/18ce844a5cf04dd5a148a2817280720e 2024-12-07T15:14:59,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/A/34d79872ad7c4eac904612021c48f6e7 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/34d79872ad7c4eac904612021c48f6e7 2024-12-07T15:14:59,984 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/34d79872ad7c4eac904612021c48f6e7, entries=150, sequenceid=737, filesize=12.0 K 2024-12-07T15:14:59,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/B/09caafc2f44743bf88a7fc49d6d63665 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/09caafc2f44743bf88a7fc49d6d63665 2024-12-07T15:14:59,988 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/09caafc2f44743bf88a7fc49d6d63665, entries=150, sequenceid=737, filesize=12.0 K 2024-12-07T15:14:59,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/C/18ce844a5cf04dd5a148a2817280720e as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/18ce844a5cf04dd5a148a2817280720e 2024-12-07T15:14:59,992 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/18ce844a5cf04dd5a148a2817280720e, entries=150, sequenceid=737, filesize=12.0 K 2024-12-07T15:14:59,992 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=20.13 KB/20610 for 3cb6bfad51268ddac87316028a55f638 in 862ms, sequenceid=737, compaction requested=false 2024-12-07T15:14:59,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2603): Flush status journal for 3cb6bfad51268ddac87316028a55f638: 2024-12-07T15:14:59,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:14:59,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=87 2024-12-07T15:14:59,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4169): Remote procedure done, pid=87 2024-12-07T15:14:59,995 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-12-07T15:14:59,995 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7880 sec 2024-12-07T15:14:59,996 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees in 1.7920 sec 2024-12-07T15:15:00,004 INFO [Thread-1091 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1091. 2024-12-07T15:15:00,004 DEBUG [Thread-1091 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T15:15:00,004 DEBUG [Thread-1091 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:15:00,004 DEBUG [Thread-1091 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:15:00,004 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T15:15:00,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-12-07T15:15:00,354 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T15:15:00,475 INFO [Thread-1089 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1089. 2024-12-07T15:15:00,475 DEBUG [Thread-1089 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T15:15:00,475 DEBUG [Thread-1089 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:15:00,475 DEBUG [Thread-1089 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:15:00,475 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T15:15:00,475 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-07T15:15:00,475 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 136 2024-12-07T15:15:00,475 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 120 2024-12-07T15:15:00,475 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 124 2024-12-07T15:15:00,475 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 124 2024-12-07T15:15:00,475 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 132 2024-12-07T15:15:00,475 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-07T15:15:00,475 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2793 2024-12-07T15:15:00,475 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2639 2024-12-07T15:15:00,475 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2641 2024-12-07T15:15:00,475 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2786 2024-12-07T15:15:00,475 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2647 2024-12-07T15:15:00,475 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-07T15:15:00,476 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-07T15:15:00,476 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-07T15:15:00,476 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:15:00,476 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:15:00,476 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T15:15:00,477 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-07T15:15:00,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] procedure2.ProcedureExecutor(1139): Stored pid=88, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-07T15:15:00,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-12-07T15:15:00,480 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733584500480"}]},"ts":"1733584500480"} 2024-12-07T15:15:00,482 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-07T15:15:00,482 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-07T15:15:00,483 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-07T15:15:00,485 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3cb6bfad51268ddac87316028a55f638, UNASSIGN}] 2024-12-07T15:15:00,486 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3cb6bfad51268ddac87316028a55f638, UNASSIGN 2024-12-07T15:15:00,486 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=3cb6bfad51268ddac87316028a55f638, regionState=CLOSING, regionLocation=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:00,488 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3cb6bfad51268ddac87316028a55f638, UNASSIGN because future has completed 2024-12-07T15:15:00,488 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-07T15:15:00,488 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401}] 2024-12-07T15:15:00,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-12-07T15:15:00,641 INFO [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(122): Close 3cb6bfad51268ddac87316028a55f638 2024-12-07T15:15:00,641 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-07T15:15:00,642 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1722): Closing 3cb6bfad51268ddac87316028a55f638, disabling compactions & flushes 2024-12-07T15:15:00,642 INFO [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:15:00,642 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:15:00,642 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. after waiting 0 ms 2024-12-07T15:15:00,642 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:15:00,642 INFO [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(2902): Flushing 3cb6bfad51268ddac87316028a55f638 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-07T15:15:00,642 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3cb6bfad51268ddac87316028a55f638, store=A 2024-12-07T15:15:00,642 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:15:00,642 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3cb6bfad51268ddac87316028a55f638, store=B 2024-12-07T15:15:00,642 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:15:00,642 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3cb6bfad51268ddac87316028a55f638, store=C 2024-12-07T15:15:00,642 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:15:00,646 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/A/e5ac19a6785e4f4683715ffbe37ba693 is 50, key is test_row_0/A:col10/1733584499704/Put/seqid=0 2024-12-07T15:15:00,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742165_1341 (size=9857) 2024-12-07T15:15:00,661 INFO [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=748 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/A/e5ac19a6785e4f4683715ffbe37ba693 2024-12-07T15:15:00,669 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/B/421cfacbbb7a4245a4d6885582ac4d06 is 50, key is test_row_0/B:col10/1733584499704/Put/seqid=0 2024-12-07T15:15:00,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742166_1342 (size=9857) 2024-12-07T15:15:00,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-12-07T15:15:01,075 INFO [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=748 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/B/421cfacbbb7a4245a4d6885582ac4d06 2024-12-07T15:15:01,084 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/C/18b5039ea07d4315b4dc7fa8d4a40cca is 50, key is test_row_0/C:col10/1733584499704/Put/seqid=0 2024-12-07T15:15:01,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742167_1343 (size=9857) 2024-12-07T15:15:01,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-12-07T15:15:01,488 INFO [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=748 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/C/18b5039ea07d4315b4dc7fa8d4a40cca 2024-12-07T15:15:01,492 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/A/e5ac19a6785e4f4683715ffbe37ba693 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/e5ac19a6785e4f4683715ffbe37ba693 2024-12-07T15:15:01,495 INFO [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/e5ac19a6785e4f4683715ffbe37ba693, entries=100, sequenceid=748, filesize=9.6 K 2024-12-07T15:15:01,496 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/B/421cfacbbb7a4245a4d6885582ac4d06 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/421cfacbbb7a4245a4d6885582ac4d06 2024-12-07T15:15:01,499 INFO [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/421cfacbbb7a4245a4d6885582ac4d06, entries=100, sequenceid=748, filesize=9.6 K 2024-12-07T15:15:01,499 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/.tmp/C/18b5039ea07d4315b4dc7fa8d4a40cca as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/18b5039ea07d4315b4dc7fa8d4a40cca 2024-12-07T15:15:01,502 INFO [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/18b5039ea07d4315b4dc7fa8d4a40cca, entries=100, sequenceid=748, filesize=9.6 K 2024-12-07T15:15:01,503 INFO [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 3cb6bfad51268ddac87316028a55f638 in 861ms, sequenceid=748, compaction requested=true 2024-12-07T15:15:01,503 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/b419ed8660bf411d884e1c34ac8dab96, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/1613379d99c64ecdb95a3592025a1cd3, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/474a97542ab447028f9bf47484116cb0, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/5d14fc84f2c440c2abe9f767050c001d, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/9e4ce26322904875b1d98fc73ec4a212, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/71368f30be044cd0a11ecf442aa836c3, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/74679813d7844c048189112e297e283a, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/0c8ea77ed2de4a3a980c0f9289645bb3, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/25764b046b2b44588043c04765c36b27, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/191ba8c3737840dd9273b7a64febd3a2, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/0b59732e60304f4d9d2bcb8cd7f54d87, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/cb77b159166d4ae2b6a1d3b285332ce5, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/8c38c9c808ce43a2901bd52521f9b574, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/70f86713795b4cfd8a4bb3b9539e2249, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/b677a6ebc536470e8c68520786896af6, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/648e626a95c64d6b970fe044fb1926c7, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/769d23839966476c8aa7c76f44a78e74, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/9b04d341ea4141c984361d4a48fc5bb2, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/b942999ede43487f9e5f24035c6f6338, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/c866b0fb764442aa913c2d804c211724, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/30fc2ee5b5744b5ab9a1c92027fdc6f6, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/8f152eff200b4ad4995940d9f1fc0360, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/17fde2ae407a47fa96dbc6da7bab1ba6, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/1dedfeb0e4984ce4859d3f659615d720, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/8b5565f12da448cca237e52dc244be0b, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/b4a665bee89049de8eb4f65ee164f182, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/aebbe70aa7c842d6ad56790a78c0b508, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/e101b0be2a834dc9ba26773b2f16b212, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/57f2a447abd94af488155ba5b9c51db8, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/6e50cc150dc24620bfe0235cab12e6ac, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/0db05f8288a7406694d5b3ff06a15491, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/56734f24de90472d9c32c43d8b41ff23, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/77f90b60377a4f6292ff6f1d51739d4f, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/0e8b71e8ce8b4e7b8dd6a0a677b6fdfc] to archive 2024-12-07T15:15:01,504 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-07T15:15:01,506 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/b419ed8660bf411d884e1c34ac8dab96 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/b419ed8660bf411d884e1c34ac8dab96 2024-12-07T15:15:01,506 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/1613379d99c64ecdb95a3592025a1cd3 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/1613379d99c64ecdb95a3592025a1cd3 2024-12-07T15:15:01,507 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/474a97542ab447028f9bf47484116cb0 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/474a97542ab447028f9bf47484116cb0 2024-12-07T15:15:01,508 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/5d14fc84f2c440c2abe9f767050c001d to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/5d14fc84f2c440c2abe9f767050c001d 2024-12-07T15:15:01,509 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/9e4ce26322904875b1d98fc73ec4a212 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/9e4ce26322904875b1d98fc73ec4a212 2024-12-07T15:15:01,510 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/71368f30be044cd0a11ecf442aa836c3 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/71368f30be044cd0a11ecf442aa836c3 2024-12-07T15:15:01,511 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/74679813d7844c048189112e297e283a to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/74679813d7844c048189112e297e283a 2024-12-07T15:15:01,512 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/0c8ea77ed2de4a3a980c0f9289645bb3 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/0c8ea77ed2de4a3a980c0f9289645bb3 2024-12-07T15:15:01,513 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/25764b046b2b44588043c04765c36b27 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/25764b046b2b44588043c04765c36b27 2024-12-07T15:15:01,514 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/191ba8c3737840dd9273b7a64febd3a2 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/191ba8c3737840dd9273b7a64febd3a2 2024-12-07T15:15:01,515 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/0b59732e60304f4d9d2bcb8cd7f54d87 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/0b59732e60304f4d9d2bcb8cd7f54d87 2024-12-07T15:15:01,516 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/cb77b159166d4ae2b6a1d3b285332ce5 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/cb77b159166d4ae2b6a1d3b285332ce5 2024-12-07T15:15:01,517 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/8c38c9c808ce43a2901bd52521f9b574 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/8c38c9c808ce43a2901bd52521f9b574 2024-12-07T15:15:01,518 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/70f86713795b4cfd8a4bb3b9539e2249 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/70f86713795b4cfd8a4bb3b9539e2249 2024-12-07T15:15:01,519 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/b677a6ebc536470e8c68520786896af6 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/b677a6ebc536470e8c68520786896af6 2024-12-07T15:15:01,520 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/648e626a95c64d6b970fe044fb1926c7 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/648e626a95c64d6b970fe044fb1926c7 2024-12-07T15:15:01,521 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/769d23839966476c8aa7c76f44a78e74 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/769d23839966476c8aa7c76f44a78e74 2024-12-07T15:15:01,522 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/9b04d341ea4141c984361d4a48fc5bb2 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/9b04d341ea4141c984361d4a48fc5bb2 2024-12-07T15:15:01,523 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/b942999ede43487f9e5f24035c6f6338 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/b942999ede43487f9e5f24035c6f6338 2024-12-07T15:15:01,524 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/c866b0fb764442aa913c2d804c211724 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/c866b0fb764442aa913c2d804c211724 2024-12-07T15:15:01,525 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/30fc2ee5b5744b5ab9a1c92027fdc6f6 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/30fc2ee5b5744b5ab9a1c92027fdc6f6 2024-12-07T15:15:01,526 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/8f152eff200b4ad4995940d9f1fc0360 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/8f152eff200b4ad4995940d9f1fc0360 2024-12-07T15:15:01,527 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/17fde2ae407a47fa96dbc6da7bab1ba6 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/17fde2ae407a47fa96dbc6da7bab1ba6 2024-12-07T15:15:01,528 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/1dedfeb0e4984ce4859d3f659615d720 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/1dedfeb0e4984ce4859d3f659615d720 2024-12-07T15:15:01,529 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/8b5565f12da448cca237e52dc244be0b to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/8b5565f12da448cca237e52dc244be0b 2024-12-07T15:15:01,530 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/b4a665bee89049de8eb4f65ee164f182 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/b4a665bee89049de8eb4f65ee164f182 2024-12-07T15:15:01,531 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/aebbe70aa7c842d6ad56790a78c0b508 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/aebbe70aa7c842d6ad56790a78c0b508 2024-12-07T15:15:01,532 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/e101b0be2a834dc9ba26773b2f16b212 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/e101b0be2a834dc9ba26773b2f16b212 2024-12-07T15:15:01,533 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/57f2a447abd94af488155ba5b9c51db8 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/57f2a447abd94af488155ba5b9c51db8 2024-12-07T15:15:01,534 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/6e50cc150dc24620bfe0235cab12e6ac to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/6e50cc150dc24620bfe0235cab12e6ac 2024-12-07T15:15:01,536 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/0db05f8288a7406694d5b3ff06a15491 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/0db05f8288a7406694d5b3ff06a15491 2024-12-07T15:15:01,537 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/56734f24de90472d9c32c43d8b41ff23 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/56734f24de90472d9c32c43d8b41ff23 2024-12-07T15:15:01,537 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/77f90b60377a4f6292ff6f1d51739d4f to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/77f90b60377a4f6292ff6f1d51739d4f 2024-12-07T15:15:01,538 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/0e8b71e8ce8b4e7b8dd6a0a677b6fdfc to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/0e8b71e8ce8b4e7b8dd6a0a677b6fdfc 2024-12-07T15:15:01,541 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/d3b97759282e41e78d6e2be2d0d29067, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/4e0b7591f37b4ef8b7cf6eebd164e98d, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/cc06d1fdb66442e1a8b5232e296c7a8d, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/9931ec432ead44278cb4b298f80e6c2e, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/984d44aca1604cc18f0ca6eb2d0fa3c5, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/51c092a9bf3448b6a3a09018e329481e, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/3cde1d755e154205bedb46845d8d57ce, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/5f463b7d437c4bbdbec72beb4fc69d4d, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/530286df9c77447ba86614bc73e2ce13, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/69057901c6d345c9821dbd22ea0a360c, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/2145644a542145b4ad17d0b921aabab6, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/85e8db7655ac4e57baf53ad04fe77b1d, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/976c3184f3ec4cd49d94571770fbc46c, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/e69c02b9c1ce44ee8f5e958b1d53427f, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/3cedb7b17be142b4bc2cf7fff8c2fa2b, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/9235aedbfba14158b9ce28d40428fdd3, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/0d37ea2c1bc84e64b6187b437f422ba3, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/b4852d986724445abcf9474be4495cbe, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/1690aec66e2a4aea9cc5cdf470f89a15, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/721e7e982da946d7a05009a0b72e1a8f, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/87f571fc9fc1404e8171fccb95870845, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/f87f547710c9435a9bbdb5f60d736927, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/08a487c1b672422ab4fc878f67a3bfd5, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/9c2253b642c24f7eaf3f025fbe64750c, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/6f193ef207204ac786d860f531fdc4ca, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/75b005e0c6364fb8900cc6571f8ebb1d, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/deef37995e404e629a75549b44c7ca42, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/aa43990985f34eeca848ff3e74071ac2, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/3833f9658d2a4893922f57a58b872bbd, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/823c2a4923c54692aa583b4b901067ea, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/d46717666e714973a2536cb9922cbb78, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/57592bd7391b444f9e217d3b38b290bd, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/6e911842dcde4c3f83afdaeedc48acb8, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/d63e87a1d3424925b6e07c179df2eb2f] to archive 2024-12-07T15:15:01,541 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-07T15:15:01,542 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/d3b97759282e41e78d6e2be2d0d29067 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/d3b97759282e41e78d6e2be2d0d29067 2024-12-07T15:15:01,543 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/4e0b7591f37b4ef8b7cf6eebd164e98d to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/4e0b7591f37b4ef8b7cf6eebd164e98d 2024-12-07T15:15:01,544 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/cc06d1fdb66442e1a8b5232e296c7a8d to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/cc06d1fdb66442e1a8b5232e296c7a8d 2024-12-07T15:15:01,545 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/9931ec432ead44278cb4b298f80e6c2e to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/9931ec432ead44278cb4b298f80e6c2e 2024-12-07T15:15:01,546 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/984d44aca1604cc18f0ca6eb2d0fa3c5 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/984d44aca1604cc18f0ca6eb2d0fa3c5 2024-12-07T15:15:01,547 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/51c092a9bf3448b6a3a09018e329481e to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/51c092a9bf3448b6a3a09018e329481e 2024-12-07T15:15:01,547 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/3cde1d755e154205bedb46845d8d57ce to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/3cde1d755e154205bedb46845d8d57ce 2024-12-07T15:15:01,548 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/5f463b7d437c4bbdbec72beb4fc69d4d to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/5f463b7d437c4bbdbec72beb4fc69d4d 2024-12-07T15:15:01,549 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/530286df9c77447ba86614bc73e2ce13 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/530286df9c77447ba86614bc73e2ce13 2024-12-07T15:15:01,550 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/69057901c6d345c9821dbd22ea0a360c to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/69057901c6d345c9821dbd22ea0a360c 2024-12-07T15:15:01,551 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/2145644a542145b4ad17d0b921aabab6 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/2145644a542145b4ad17d0b921aabab6 2024-12-07T15:15:01,552 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/85e8db7655ac4e57baf53ad04fe77b1d to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/85e8db7655ac4e57baf53ad04fe77b1d 2024-12-07T15:15:01,554 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/976c3184f3ec4cd49d94571770fbc46c to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/976c3184f3ec4cd49d94571770fbc46c 2024-12-07T15:15:01,555 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/e69c02b9c1ce44ee8f5e958b1d53427f to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/e69c02b9c1ce44ee8f5e958b1d53427f 2024-12-07T15:15:01,556 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/3cedb7b17be142b4bc2cf7fff8c2fa2b to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/3cedb7b17be142b4bc2cf7fff8c2fa2b 2024-12-07T15:15:01,557 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/9235aedbfba14158b9ce28d40428fdd3 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/9235aedbfba14158b9ce28d40428fdd3 2024-12-07T15:15:01,558 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/0d37ea2c1bc84e64b6187b437f422ba3 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/0d37ea2c1bc84e64b6187b437f422ba3 2024-12-07T15:15:01,559 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/b4852d986724445abcf9474be4495cbe to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/b4852d986724445abcf9474be4495cbe 2024-12-07T15:15:01,559 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/1690aec66e2a4aea9cc5cdf470f89a15 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/1690aec66e2a4aea9cc5cdf470f89a15 2024-12-07T15:15:01,560 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/721e7e982da946d7a05009a0b72e1a8f to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/721e7e982da946d7a05009a0b72e1a8f 2024-12-07T15:15:01,561 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/87f571fc9fc1404e8171fccb95870845 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/87f571fc9fc1404e8171fccb95870845 2024-12-07T15:15:01,562 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/f87f547710c9435a9bbdb5f60d736927 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/f87f547710c9435a9bbdb5f60d736927 2024-12-07T15:15:01,563 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/08a487c1b672422ab4fc878f67a3bfd5 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/08a487c1b672422ab4fc878f67a3bfd5 2024-12-07T15:15:01,564 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/9c2253b642c24f7eaf3f025fbe64750c to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/9c2253b642c24f7eaf3f025fbe64750c 2024-12-07T15:15:01,565 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/6f193ef207204ac786d860f531fdc4ca to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/6f193ef207204ac786d860f531fdc4ca 2024-12-07T15:15:01,566 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/75b005e0c6364fb8900cc6571f8ebb1d to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/75b005e0c6364fb8900cc6571f8ebb1d 2024-12-07T15:15:01,567 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/deef37995e404e629a75549b44c7ca42 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/deef37995e404e629a75549b44c7ca42 2024-12-07T15:15:01,568 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/aa43990985f34eeca848ff3e74071ac2 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/aa43990985f34eeca848ff3e74071ac2 2024-12-07T15:15:01,569 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/3833f9658d2a4893922f57a58b872bbd to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/3833f9658d2a4893922f57a58b872bbd 2024-12-07T15:15:01,570 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/823c2a4923c54692aa583b4b901067ea to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/823c2a4923c54692aa583b4b901067ea 2024-12-07T15:15:01,571 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/d46717666e714973a2536cb9922cbb78 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/d46717666e714973a2536cb9922cbb78 2024-12-07T15:15:01,571 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/57592bd7391b444f9e217d3b38b290bd to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/57592bd7391b444f9e217d3b38b290bd 2024-12-07T15:15:01,572 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/6e911842dcde4c3f83afdaeedc48acb8 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/6e911842dcde4c3f83afdaeedc48acb8 2024-12-07T15:15:01,573 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/d63e87a1d3424925b6e07c179df2eb2f to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/d63e87a1d3424925b6e07c179df2eb2f 2024-12-07T15:15:01,575 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/cb9b93545cfe4ce38cce242439564f52, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/f67e7ab9b95d430dbce9903449cb4c47, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/c44710a364514d968ffe3579f50a5fab, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/684491ae329b4ad79ea7f6b08d670f80, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/880e42fc933449b6afbd9e0d26facaed, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/e1861a17d0ff4f16aa0217d06b6509b1, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/33d7e474111f44558b2cc3f9e15c0ff4, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/886de3c24eb3424b83423b683c2103cd, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/16d29d662d924da3bd3c7db74f5c2c26, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/a1ebe28579b24c928bee80431d4b6ebb, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/7cc98ccf0d96421c8cdc0c19c0d9831b, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/9d4fa6ef08804d28bdc7ba361ea71ca0, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/b2034b69543e432487736057fea36458, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/5df0575bd309491e8c9d0feed602d0a1, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/ba19eda8c8f34a448f3274686c1e7a05, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/cbdc5ce731e044638cf91be3c10a7eae, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/cd659314350f40aaa16809ac7f20ed31, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/ec7315b4edc947efab7c9c11e7d6e354, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/3cd7c3dd943249608c3cfa676f72896c, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/aa5e37572c5b41dd8e2d108bf877ba93, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/a2feb325ccaa4099b9212d06bda4ec02, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/fb156a65081a445e9794ff3d5231ea82, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/d928cb287b6940058081f79629b33a07, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/7d2bf6adb31749748f173940f47c6ab0, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/56c5bf01f5f0403a8aeab06851623216, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/dfbd8ada737f4d70ad806703b13dbde7, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/b2959885ce29424691a703366c665b99, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/831a55b1048f4581b03da8a915c586ee, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/2f2e755bd3974fd39c0d9fe4c8ec4f62, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/d43d5ab6de484de6994e7b9aaa273cf4, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/d797ec72aa6c43e2a7b3cac5ac984faa, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/6ad03a78dd4249a083cd1e8162484701, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/e76cbbd8243d423c88ac370b7c109737] to archive 2024-12-07T15:15:01,576 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-07T15:15:01,577 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/cb9b93545cfe4ce38cce242439564f52 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/cb9b93545cfe4ce38cce242439564f52 2024-12-07T15:15:01,579 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/f67e7ab9b95d430dbce9903449cb4c47 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/f67e7ab9b95d430dbce9903449cb4c47 2024-12-07T15:15:01,580 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/c44710a364514d968ffe3579f50a5fab to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/c44710a364514d968ffe3579f50a5fab 2024-12-07T15:15:01,581 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/684491ae329b4ad79ea7f6b08d670f80 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/684491ae329b4ad79ea7f6b08d670f80 2024-12-07T15:15:01,582 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/880e42fc933449b6afbd9e0d26facaed to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/880e42fc933449b6afbd9e0d26facaed 2024-12-07T15:15:01,583 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/e1861a17d0ff4f16aa0217d06b6509b1 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/e1861a17d0ff4f16aa0217d06b6509b1 2024-12-07T15:15:01,584 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/33d7e474111f44558b2cc3f9e15c0ff4 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/33d7e474111f44558b2cc3f9e15c0ff4 2024-12-07T15:15:01,585 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/886de3c24eb3424b83423b683c2103cd to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/886de3c24eb3424b83423b683c2103cd 2024-12-07T15:15:01,585 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/16d29d662d924da3bd3c7db74f5c2c26 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/16d29d662d924da3bd3c7db74f5c2c26 2024-12-07T15:15:01,586 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/a1ebe28579b24c928bee80431d4b6ebb to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/a1ebe28579b24c928bee80431d4b6ebb 2024-12-07T15:15:01,587 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/7cc98ccf0d96421c8cdc0c19c0d9831b to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/7cc98ccf0d96421c8cdc0c19c0d9831b 2024-12-07T15:15:01,588 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/9d4fa6ef08804d28bdc7ba361ea71ca0 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/9d4fa6ef08804d28bdc7ba361ea71ca0 2024-12-07T15:15:01,589 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/b2034b69543e432487736057fea36458 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/b2034b69543e432487736057fea36458 2024-12-07T15:15:01,590 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/5df0575bd309491e8c9d0feed602d0a1 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/5df0575bd309491e8c9d0feed602d0a1 2024-12-07T15:15:01,591 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/ba19eda8c8f34a448f3274686c1e7a05 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/ba19eda8c8f34a448f3274686c1e7a05 2024-12-07T15:15:01,592 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/cbdc5ce731e044638cf91be3c10a7eae to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/cbdc5ce731e044638cf91be3c10a7eae 2024-12-07T15:15:01,593 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/cd659314350f40aaa16809ac7f20ed31 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/cd659314350f40aaa16809ac7f20ed31 2024-12-07T15:15:01,594 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/ec7315b4edc947efab7c9c11e7d6e354 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/ec7315b4edc947efab7c9c11e7d6e354 2024-12-07T15:15:01,595 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/3cd7c3dd943249608c3cfa676f72896c to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/3cd7c3dd943249608c3cfa676f72896c 2024-12-07T15:15:01,596 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/aa5e37572c5b41dd8e2d108bf877ba93 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/aa5e37572c5b41dd8e2d108bf877ba93 2024-12-07T15:15:01,597 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/a2feb325ccaa4099b9212d06bda4ec02 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/a2feb325ccaa4099b9212d06bda4ec02 2024-12-07T15:15:01,598 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/fb156a65081a445e9794ff3d5231ea82 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/fb156a65081a445e9794ff3d5231ea82 2024-12-07T15:15:01,599 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/d928cb287b6940058081f79629b33a07 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/d928cb287b6940058081f79629b33a07 2024-12-07T15:15:01,601 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/7d2bf6adb31749748f173940f47c6ab0 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/7d2bf6adb31749748f173940f47c6ab0 2024-12-07T15:15:01,602 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/56c5bf01f5f0403a8aeab06851623216 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/56c5bf01f5f0403a8aeab06851623216 2024-12-07T15:15:01,603 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/dfbd8ada737f4d70ad806703b13dbde7 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/dfbd8ada737f4d70ad806703b13dbde7 2024-12-07T15:15:01,604 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/b2959885ce29424691a703366c665b99 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/b2959885ce29424691a703366c665b99 2024-12-07T15:15:01,605 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/831a55b1048f4581b03da8a915c586ee to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/831a55b1048f4581b03da8a915c586ee 2024-12-07T15:15:01,606 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/2f2e755bd3974fd39c0d9fe4c8ec4f62 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/2f2e755bd3974fd39c0d9fe4c8ec4f62 2024-12-07T15:15:01,607 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/d43d5ab6de484de6994e7b9aaa273cf4 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/d43d5ab6de484de6994e7b9aaa273cf4 2024-12-07T15:15:01,608 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/d797ec72aa6c43e2a7b3cac5ac984faa to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/d797ec72aa6c43e2a7b3cac5ac984faa 2024-12-07T15:15:01,609 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/6ad03a78dd4249a083cd1e8162484701 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/6ad03a78dd4249a083cd1e8162484701 2024-12-07T15:15:01,610 DEBUG [StoreCloser-TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/e76cbbd8243d423c88ac370b7c109737 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/e76cbbd8243d423c88ac370b7c109737 2024-12-07T15:15:01,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-12-07T15:15:01,614 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/recovered.edits/751.seqid, newMaxSeqId=751, maxSeqId=1 2024-12-07T15:15:01,615 INFO [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. 2024-12-07T15:15:01,615 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1676): Region close journal for 3cb6bfad51268ddac87316028a55f638: Waiting for close lock at 1733584500641Running coprocessor pre-close hooks at 1733584500641Disabling compacts and flushes for region at 1733584500641Disabling writes for close at 1733584500642 (+1 ms)Obtaining lock to block concurrent updates at 1733584500642Preparing flush snapshotting stores in 3cb6bfad51268ddac87316028a55f638 at 1733584500642Finished memstore snapshotting TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1733584500642Flushing stores of TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638. at 1733584500643 (+1 ms)Flushing 3cb6bfad51268ddac87316028a55f638/A: creating writer at 1733584500643Flushing 3cb6bfad51268ddac87316028a55f638/A: appending metadata at 1733584500646 (+3 ms)Flushing 3cb6bfad51268ddac87316028a55f638/A: closing flushed file at 1733584500646Flushing 3cb6bfad51268ddac87316028a55f638/B: creating writer at 1733584500666 (+20 ms)Flushing 3cb6bfad51268ddac87316028a55f638/B: appending metadata at 1733584500668 (+2 ms)Flushing 3cb6bfad51268ddac87316028a55f638/B: closing flushed file at 1733584500669 (+1 ms)Flushing 3cb6bfad51268ddac87316028a55f638/C: creating writer at 1733584501080 (+411 ms)Flushing 3cb6bfad51268ddac87316028a55f638/C: appending metadata at 1733584501083 (+3 ms)Flushing 3cb6bfad51268ddac87316028a55f638/C: closing flushed file at 1733584501083Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@13f55c22: reopening flushed file at 1733584501491 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7524ab01: reopening flushed file at 1733584501495 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@57a9fbee: reopening flushed file at 1733584501499 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 3cb6bfad51268ddac87316028a55f638 in 861ms, sequenceid=748, compaction requested=true at 1733584501503 (+4 ms)Writing region close event to WAL at 1733584501611 (+108 ms)Running coprocessor post-close hooks at 1733584501615 (+4 ms)Closed at 1733584501615 2024-12-07T15:15:01,617 INFO [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(157): Closed 3cb6bfad51268ddac87316028a55f638 2024-12-07T15:15:01,618 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=3cb6bfad51268ddac87316028a55f638, regionState=CLOSED 2024-12-07T15:15:01,619 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=91, ppid=90, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 because future has completed 2024-12-07T15:15:01,622 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-12-07T15:15:01,622 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; CloseRegionProcedure 3cb6bfad51268ddac87316028a55f638, server=d6797ebe9c52,46009,1733584418401 in 1.1320 sec 2024-12-07T15:15:01,624 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-12-07T15:15:01,624 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3cb6bfad51268ddac87316028a55f638, UNASSIGN in 1.1380 sec 2024-12-07T15:15:01,626 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-12-07T15:15:01,626 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.1410 sec 2024-12-07T15:15:01,628 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733584501628"}]},"ts":"1733584501628"} 2024-12-07T15:15:01,629 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-07T15:15:01,629 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-07T15:15:01,631 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.1530 sec 2024-12-07T15:15:02,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-12-07T15:15:02,623 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-07T15:15:02,624 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-07T15:15:02,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] procedure2.ProcedureExecutor(1139): Stored pid=92, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-07T15:15:02,626 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=92, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-07T15:15:02,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-12-07T15:15:02,627 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=92, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-07T15:15:02,630 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638 2024-12-07T15:15:02,633 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A, FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B, FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C, FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/recovered.edits] 2024-12-07T15:15:02,639 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/0dda79aa85764d358ca0d7c5546cf553 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/0dda79aa85764d358ca0d7c5546cf553 2024-12-07T15:15:02,640 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/34d79872ad7c4eac904612021c48f6e7 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/34d79872ad7c4eac904612021c48f6e7 2024-12-07T15:15:02,642 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/e5ac19a6785e4f4683715ffbe37ba693 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/A/e5ac19a6785e4f4683715ffbe37ba693 2024-12-07T15:15:02,644 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/0221994b2a4d43168fdce5a12fa3603c to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/0221994b2a4d43168fdce5a12fa3603c 2024-12-07T15:15:02,647 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/09caafc2f44743bf88a7fc49d6d63665 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/09caafc2f44743bf88a7fc49d6d63665 2024-12-07T15:15:02,653 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/421cfacbbb7a4245a4d6885582ac4d06 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/B/421cfacbbb7a4245a4d6885582ac4d06 2024-12-07T15:15:02,658 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/18b5039ea07d4315b4dc7fa8d4a40cca to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/18b5039ea07d4315b4dc7fa8d4a40cca 2024-12-07T15:15:02,662 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/18ce844a5cf04dd5a148a2817280720e to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/18ce844a5cf04dd5a148a2817280720e 2024-12-07T15:15:02,667 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/ef68d7503ec046599559888b6cbcd2ad to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/C/ef68d7503ec046599559888b6cbcd2ad 2024-12-07T15:15:02,670 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/recovered.edits/751.seqid to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638/recovered.edits/751.seqid 2024-12-07T15:15:02,671 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/3cb6bfad51268ddac87316028a55f638 2024-12-07T15:15:02,671 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-07T15:15:02,674 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=92, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-07T15:15:02,678 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-07T15:15:02,680 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-07T15:15:02,682 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=92, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-07T15:15:02,682 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-07T15:15:02,682 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733584502682"}]},"ts":"9223372036854775807"} 2024-12-07T15:15:02,684 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-07T15:15:02,684 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 3cb6bfad51268ddac87316028a55f638, NAME => 'TestAcidGuarantees,,1733584476659.3cb6bfad51268ddac87316028a55f638.', STARTKEY => '', ENDKEY => ''}] 2024-12-07T15:15:02,684 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-07T15:15:02,684 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733584502684"}]},"ts":"9223372036854775807"} 2024-12-07T15:15:02,687 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-07T15:15:02,687 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=92, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-07T15:15:02,688 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 63 msec 2024-12-07T15:15:02,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-12-07T15:15:02,733 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-07T15:15:02,733 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-07T15:15:02,745 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=250 (was 251), OpenFileDescriptor=453 (was 457), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=512 (was 433) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7622 (was 7362) - AvailableMemoryMB LEAK? - 2024-12-07T15:15:02,756 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=250, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=512, ProcessCount=11, AvailableMemoryMB=7622 2024-12-07T15:15:02,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-07T15:15:02,757 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-07T15:15:02,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-07T15:15:02,759 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-07T15:15:02,759 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:02,759 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 93 2024-12-07T15:15:02,760 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-07T15:15:02,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-07T15:15:02,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742168_1344 (size=1061) 2024-12-07T15:15:02,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-07T15:15:03,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-07T15:15:03,167 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd 2024-12-07T15:15:03,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742169_1345 (size=53) 2024-12-07T15:15:03,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-07T15:15:03,573 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T15:15:03,573 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing c0d1e6963b45fdb2c1b5de83c5d92b34, disabling compactions & flushes 2024-12-07T15:15:03,573 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:03,573 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:03,573 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. after waiting 0 ms 2024-12-07T15:15:03,573 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:03,573 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:03,573 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for c0d1e6963b45fdb2c1b5de83c5d92b34: Waiting for close lock at 1733584503573Disabling compacts and flushes for region at 1733584503573Disabling writes for close at 1733584503573Writing region close event to WAL at 1733584503573Closed at 1733584503573 2024-12-07T15:15:03,574 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-07T15:15:03,575 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733584503574"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733584503574"}]},"ts":"1733584503574"} 2024-12-07T15:15:03,576 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-07T15:15:03,577 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-07T15:15:03,577 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733584503577"}]},"ts":"1733584503577"} 2024-12-07T15:15:03,578 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-07T15:15:03,579 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c0d1e6963b45fdb2c1b5de83c5d92b34, ASSIGN}] 2024-12-07T15:15:03,580 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c0d1e6963b45fdb2c1b5de83c5d92b34, ASSIGN 2024-12-07T15:15:03,581 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c0d1e6963b45fdb2c1b5de83c5d92b34, ASSIGN; state=OFFLINE, location=d6797ebe9c52,46009,1733584418401; forceNewPlan=false, retain=false 2024-12-07T15:15:03,731 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=c0d1e6963b45fdb2c1b5de83c5d92b34, regionState=OPENING, regionLocation=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:03,733 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c0d1e6963b45fdb2c1b5de83c5d92b34, ASSIGN because future has completed 2024-12-07T15:15:03,733 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=95, ppid=94, state=RUNNABLE, hasLock=false; OpenRegionProcedure c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401}] 2024-12-07T15:15:03,888 INFO [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:03,888 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(7752): Opening region: {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} 2024-12-07T15:15:03,889 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees c0d1e6963b45fdb2c1b5de83c5d92b34 2024-12-07T15:15:03,889 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T15:15:03,889 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(7794): checking encryption for c0d1e6963b45fdb2c1b5de83c5d92b34 2024-12-07T15:15:03,889 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(7797): checking classloading for c0d1e6963b45fdb2c1b5de83c5d92b34 2024-12-07T15:15:03,890 INFO [StoreOpener-c0d1e6963b45fdb2c1b5de83c5d92b34-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region c0d1e6963b45fdb2c1b5de83c5d92b34 2024-12-07T15:15:03,891 INFO [StoreOpener-c0d1e6963b45fdb2c1b5de83c5d92b34-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-07T15:15:03,892 INFO [StoreOpener-c0d1e6963b45fdb2c1b5de83c5d92b34-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c0d1e6963b45fdb2c1b5de83c5d92b34 columnFamilyName A 2024-12-07T15:15:03,892 DEBUG [StoreOpener-c0d1e6963b45fdb2c1b5de83c5d92b34-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:03,892 INFO [StoreOpener-c0d1e6963b45fdb2c1b5de83c5d92b34-1 {}] regionserver.HStore(327): Store=c0d1e6963b45fdb2c1b5de83c5d92b34/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T15:15:03,892 INFO [StoreOpener-c0d1e6963b45fdb2c1b5de83c5d92b34-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region c0d1e6963b45fdb2c1b5de83c5d92b34 2024-12-07T15:15:03,893 INFO [StoreOpener-c0d1e6963b45fdb2c1b5de83c5d92b34-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-07T15:15:03,893 INFO [StoreOpener-c0d1e6963b45fdb2c1b5de83c5d92b34-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c0d1e6963b45fdb2c1b5de83c5d92b34 columnFamilyName B 2024-12-07T15:15:03,893 DEBUG [StoreOpener-c0d1e6963b45fdb2c1b5de83c5d92b34-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:03,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-07T15:15:03,893 INFO [StoreOpener-c0d1e6963b45fdb2c1b5de83c5d92b34-1 {}] regionserver.HStore(327): Store=c0d1e6963b45fdb2c1b5de83c5d92b34/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T15:15:03,893 INFO [StoreOpener-c0d1e6963b45fdb2c1b5de83c5d92b34-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region c0d1e6963b45fdb2c1b5de83c5d92b34 2024-12-07T15:15:03,894 INFO [StoreOpener-c0d1e6963b45fdb2c1b5de83c5d92b34-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-07T15:15:03,894 INFO [StoreOpener-c0d1e6963b45fdb2c1b5de83c5d92b34-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c0d1e6963b45fdb2c1b5de83c5d92b34 columnFamilyName C 2024-12-07T15:15:03,894 DEBUG [StoreOpener-c0d1e6963b45fdb2c1b5de83c5d92b34-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:03,895 INFO [StoreOpener-c0d1e6963b45fdb2c1b5de83c5d92b34-1 {}] regionserver.HStore(327): Store=c0d1e6963b45fdb2c1b5de83c5d92b34/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T15:15:03,895 INFO [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:03,895 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1038): replaying wal for c0d1e6963b45fdb2c1b5de83c5d92b34 2024-12-07T15:15:03,896 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34 2024-12-07T15:15:03,896 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34 2024-12-07T15:15:03,897 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1048): stopping wal replay for c0d1e6963b45fdb2c1b5de83c5d92b34 2024-12-07T15:15:03,897 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1060): Cleaning up temporary data for c0d1e6963b45fdb2c1b5de83c5d92b34 2024-12-07T15:15:03,897 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-07T15:15:03,898 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1093): writing seq id for c0d1e6963b45fdb2c1b5de83c5d92b34 2024-12-07T15:15:03,900 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T15:15:03,900 INFO [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1114): Opened c0d1e6963b45fdb2c1b5de83c5d92b34; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61167876, jitterRate=-0.0885276198387146}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-07T15:15:03,900 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c0d1e6963b45fdb2c1b5de83c5d92b34 2024-12-07T15:15:03,901 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1006): Region open journal for c0d1e6963b45fdb2c1b5de83c5d92b34: Running coprocessor pre-open hook at 1733584503889Writing region info on filesystem at 1733584503889Initializing all the Stores at 1733584503890 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733584503890Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733584503890Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733584503890Cleaning up temporary data from old regions at 1733584503897 (+7 ms)Running coprocessor post-open hooks at 1733584503900 (+3 ms)Region opened successfully at 1733584503901 (+1 ms) 2024-12-07T15:15:03,901 INFO [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., pid=95, masterSystemTime=1733584503885 2024-12-07T15:15:03,903 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:03,903 INFO [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:03,904 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=c0d1e6963b45fdb2c1b5de83c5d92b34, regionState=OPEN, openSeqNum=2, regionLocation=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:03,905 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=95, ppid=94, state=RUNNABLE, hasLock=false; OpenRegionProcedure c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 because future has completed 2024-12-07T15:15:03,911 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=95, resume processing ppid=94 2024-12-07T15:15:03,911 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, ppid=94, state=SUCCESS, hasLock=false; OpenRegionProcedure c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 in 176 msec 2024-12-07T15:15:03,913 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-12-07T15:15:03,913 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c0d1e6963b45fdb2c1b5de83c5d92b34, ASSIGN in 332 msec 2024-12-07T15:15:03,913 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-07T15:15:03,913 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733584503913"}]},"ts":"1733584503913"} 2024-12-07T15:15:03,915 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-07T15:15:03,915 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-07T15:15:03,916 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1580 sec 2024-12-07T15:15:04,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-07T15:15:04,913 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-07T15:15:04,914 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7befe57f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:15:04,914 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6797ebe9c52,36915,-1 for getting cluster id 2024-12-07T15:15:04,914 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T15:15:04,915 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f8956274-ed33-4e16-82eb-d5d5911ec893' 2024-12-07T15:15:04,915 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T15:15:04,916 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f8956274-ed33-4e16-82eb-d5d5911ec893" 2024-12-07T15:15:04,916 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a9c651d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:15:04,916 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6797ebe9c52,36915,-1] 2024-12-07T15:15:04,916 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T15:15:04,916 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:15:04,917 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53046, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T15:15:04,917 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78a2c30d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:15:04,917 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T15:15:04,918 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6797ebe9c52,46009,1733584418401, seqNum=-1] 2024-12-07T15:15:04,918 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T15:15:04,919 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42062, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T15:15:04,920 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-07T15:15:04,920 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is d6797ebe9c52,36915,1733584417704 2024-12-07T15:15:04,920 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@76298d15 2024-12-07T15:15:04,921 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-07T15:15:04,921 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53058, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-07T15:15:04,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-07T15:15:04,923 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-07T15:15:04,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] procedure2.ProcedureExecutor(1139): Stored pid=96, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-07T15:15:04,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742170_1346 (size=1097) 2024-12-07T15:15:05,336 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-07T15:15:05,336 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-07T15:15:05,338 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=97, ppid=96, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-07T15:15:05,341 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c0d1e6963b45fdb2c1b5de83c5d92b34, REOPEN/MOVE}] 2024-12-07T15:15:05,342 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c0d1e6963b45fdb2c1b5de83c5d92b34, REOPEN/MOVE 2024-12-07T15:15:05,342 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=98 updating hbase:meta row=c0d1e6963b45fdb2c1b5de83c5d92b34, regionState=CLOSING, regionLocation=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:05,344 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c0d1e6963b45fdb2c1b5de83c5d92b34, REOPEN/MOVE because future has completed 2024-12-07T15:15:05,344 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-07T15:15:05,344 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=99, ppid=98, state=RUNNABLE, hasLock=false; CloseRegionProcedure c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401}] 2024-12-07T15:15:05,496 INFO [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] handler.UnassignRegionHandler(122): Close c0d1e6963b45fdb2c1b5de83c5d92b34 2024-12-07T15:15:05,496 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-07T15:15:05,496 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1722): Closing c0d1e6963b45fdb2c1b5de83c5d92b34, disabling compactions & flushes 2024-12-07T15:15:05,496 INFO [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:05,496 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:05,496 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. after waiting 0 ms 2024-12-07T15:15:05,496 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:05,500 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-07T15:15:05,500 INFO [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:05,500 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1676): Region close journal for c0d1e6963b45fdb2c1b5de83c5d92b34: Waiting for close lock at 1733584505496Running coprocessor pre-close hooks at 1733584505496Disabling compacts and flushes for region at 1733584505496Disabling writes for close at 1733584505496Writing region close event to WAL at 1733584505497 (+1 ms)Running coprocessor post-close hooks at 1733584505500 (+3 ms)Closed at 1733584505500 2024-12-07T15:15:05,501 WARN [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegionServer(3299): Not adding moved region record: c0d1e6963b45fdb2c1b5de83c5d92b34 to self. 2024-12-07T15:15:05,502 INFO [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] handler.UnassignRegionHandler(157): Closed c0d1e6963b45fdb2c1b5de83c5d92b34 2024-12-07T15:15:05,503 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=98 updating hbase:meta row=c0d1e6963b45fdb2c1b5de83c5d92b34, regionState=CLOSED 2024-12-07T15:15:05,504 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=99, ppid=98, state=RUNNABLE, hasLock=false; CloseRegionProcedure c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 because future has completed 2024-12-07T15:15:05,506 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=99, resume processing ppid=98 2024-12-07T15:15:05,506 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, ppid=98, state=SUCCESS, hasLock=false; CloseRegionProcedure c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 in 160 msec 2024-12-07T15:15:05,506 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c0d1e6963b45fdb2c1b5de83c5d92b34, REOPEN/MOVE; state=CLOSED, location=d6797ebe9c52,46009,1733584418401; forceNewPlan=false, retain=true 2024-12-07T15:15:05,657 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=98 updating hbase:meta row=c0d1e6963b45fdb2c1b5de83c5d92b34, regionState=OPENING, regionLocation=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:05,658 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c0d1e6963b45fdb2c1b5de83c5d92b34, REOPEN/MOVE because future has completed 2024-12-07T15:15:05,659 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=98, state=RUNNABLE, hasLock=false; OpenRegionProcedure c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401}] 2024-12-07T15:15:05,813 INFO [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:05,813 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(7752): Opening region: {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} 2024-12-07T15:15:05,813 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees c0d1e6963b45fdb2c1b5de83c5d92b34 2024-12-07T15:15:05,813 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T15:15:05,813 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(7794): checking encryption for c0d1e6963b45fdb2c1b5de83c5d92b34 2024-12-07T15:15:05,813 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(7797): checking classloading for c0d1e6963b45fdb2c1b5de83c5d92b34 2024-12-07T15:15:05,814 INFO [StoreOpener-c0d1e6963b45fdb2c1b5de83c5d92b34-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region c0d1e6963b45fdb2c1b5de83c5d92b34 2024-12-07T15:15:05,815 INFO [StoreOpener-c0d1e6963b45fdb2c1b5de83c5d92b34-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-07T15:15:05,815 INFO [StoreOpener-c0d1e6963b45fdb2c1b5de83c5d92b34-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c0d1e6963b45fdb2c1b5de83c5d92b34 columnFamilyName A 2024-12-07T15:15:05,817 DEBUG [StoreOpener-c0d1e6963b45fdb2c1b5de83c5d92b34-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:05,817 INFO [StoreOpener-c0d1e6963b45fdb2c1b5de83c5d92b34-1 {}] regionserver.HStore(327): Store=c0d1e6963b45fdb2c1b5de83c5d92b34/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T15:15:05,818 INFO [StoreOpener-c0d1e6963b45fdb2c1b5de83c5d92b34-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region c0d1e6963b45fdb2c1b5de83c5d92b34 2024-12-07T15:15:05,818 INFO [StoreOpener-c0d1e6963b45fdb2c1b5de83c5d92b34-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-07T15:15:05,818 INFO [StoreOpener-c0d1e6963b45fdb2c1b5de83c5d92b34-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c0d1e6963b45fdb2c1b5de83c5d92b34 columnFamilyName B 2024-12-07T15:15:05,818 DEBUG [StoreOpener-c0d1e6963b45fdb2c1b5de83c5d92b34-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:05,819 INFO [StoreOpener-c0d1e6963b45fdb2c1b5de83c5d92b34-1 {}] regionserver.HStore(327): Store=c0d1e6963b45fdb2c1b5de83c5d92b34/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T15:15:05,819 INFO [StoreOpener-c0d1e6963b45fdb2c1b5de83c5d92b34-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region c0d1e6963b45fdb2c1b5de83c5d92b34 2024-12-07T15:15:05,819 INFO [StoreOpener-c0d1e6963b45fdb2c1b5de83c5d92b34-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-07T15:15:05,819 INFO [StoreOpener-c0d1e6963b45fdb2c1b5de83c5d92b34-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c0d1e6963b45fdb2c1b5de83c5d92b34 columnFamilyName C 2024-12-07T15:15:05,819 DEBUG [StoreOpener-c0d1e6963b45fdb2c1b5de83c5d92b34-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:05,820 INFO [StoreOpener-c0d1e6963b45fdb2c1b5de83c5d92b34-1 {}] regionserver.HStore(327): Store=c0d1e6963b45fdb2c1b5de83c5d92b34/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T15:15:05,820 INFO [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:05,820 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1038): replaying wal for c0d1e6963b45fdb2c1b5de83c5d92b34 2024-12-07T15:15:05,821 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34 2024-12-07T15:15:05,822 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34 2024-12-07T15:15:05,823 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1048): stopping wal replay for c0d1e6963b45fdb2c1b5de83c5d92b34 2024-12-07T15:15:05,823 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1060): Cleaning up temporary data for c0d1e6963b45fdb2c1b5de83c5d92b34 2024-12-07T15:15:05,823 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-07T15:15:05,825 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1093): writing seq id for c0d1e6963b45fdb2c1b5de83c5d92b34 2024-12-07T15:15:05,825 INFO [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1114): Opened c0d1e6963b45fdb2c1b5de83c5d92b34; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66682318, jitterRate=-0.006356030702590942}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-07T15:15:05,826 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c0d1e6963b45fdb2c1b5de83c5d92b34 2024-12-07T15:15:05,826 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1006): Region open journal for c0d1e6963b45fdb2c1b5de83c5d92b34: Running coprocessor pre-open hook at 1733584505813Writing region info on filesystem at 1733584505813Initializing all the Stores at 1733584505814 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733584505814Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733584505814Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733584505814Cleaning up temporary data from old regions at 1733584505823 (+9 ms)Running coprocessor post-open hooks at 1733584505826 (+3 ms)Region opened successfully at 1733584505826 2024-12-07T15:15:05,827 INFO [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., pid=100, masterSystemTime=1733584505810 2024-12-07T15:15:05,829 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:05,829 INFO [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:05,830 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=98 updating hbase:meta row=c0d1e6963b45fdb2c1b5de83c5d92b34, regionState=OPEN, openSeqNum=5, regionLocation=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:05,831 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=100, ppid=98, state=RUNNABLE, hasLock=false; OpenRegionProcedure c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 because future has completed 2024-12-07T15:15:05,834 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=98 2024-12-07T15:15:05,834 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=98, state=SUCCESS, hasLock=false; OpenRegionProcedure c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 in 174 msec 2024-12-07T15:15:05,836 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-12-07T15:15:05,836 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c0d1e6963b45fdb2c1b5de83c5d92b34, REOPEN/MOVE in 493 msec 2024-12-07T15:15:05,838 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=97, resume processing ppid=96 2024-12-07T15:15:05,838 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, ppid=96, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 499 msec 2024-12-07T15:15:05,841 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 916 msec 2024-12-07T15:15:05,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=96 2024-12-07T15:15:05,842 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-07T15:15:05,843 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b10b9e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:15:05,843 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6797ebe9c52,36915,-1 for getting cluster id 2024-12-07T15:15:05,843 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T15:15:05,844 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f8956274-ed33-4e16-82eb-d5d5911ec893' 2024-12-07T15:15:05,844 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T15:15:05,844 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f8956274-ed33-4e16-82eb-d5d5911ec893" 2024-12-07T15:15:05,844 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d487d27, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:15:05,844 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6797ebe9c52,36915,-1] 2024-12-07T15:15:05,844 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T15:15:05,845 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:15:05,845 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53076, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T15:15:05,846 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b94eaa5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:15:05,846 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21c265f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:15:05,846 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6797ebe9c52,36915,-1 for getting cluster id 2024-12-07T15:15:05,846 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T15:15:05,847 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f8956274-ed33-4e16-82eb-d5d5911ec893' 2024-12-07T15:15:05,847 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T15:15:05,847 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f8956274-ed33-4e16-82eb-d5d5911ec893" 2024-12-07T15:15:05,847 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d7dffbb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:15:05,848 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6797ebe9c52,36915,-1] 2024-12-07T15:15:05,848 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T15:15:05,848 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:15:05,849 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53090, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T15:15:05,849 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a1d1d27, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:15:05,850 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c21b87a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:15:05,850 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6797ebe9c52,36915,-1 for getting cluster id 2024-12-07T15:15:05,850 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T15:15:05,851 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f8956274-ed33-4e16-82eb-d5d5911ec893' 2024-12-07T15:15:05,851 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T15:15:05,851 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f8956274-ed33-4e16-82eb-d5d5911ec893" 2024-12-07T15:15:05,851 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62ffa2d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:15:05,851 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6797ebe9c52,36915,-1] 2024-12-07T15:15:05,851 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T15:15:05,851 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:15:05,852 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53114, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T15:15:05,853 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a6adb54, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:15:05,853 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35e136ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:15:05,853 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6797ebe9c52,36915,-1 for getting cluster id 2024-12-07T15:15:05,853 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T15:15:05,854 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f8956274-ed33-4e16-82eb-d5d5911ec893' 2024-12-07T15:15:05,854 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T15:15:05,854 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f8956274-ed33-4e16-82eb-d5d5911ec893" 2024-12-07T15:15:05,854 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f051e39, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:15:05,854 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6797ebe9c52,36915,-1] 2024-12-07T15:15:05,854 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T15:15:05,854 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:15:05,855 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53142, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T15:15:05,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e891ce4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:15:05,856 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f004f7c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:15:05,856 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6797ebe9c52,36915,-1 for getting cluster id 2024-12-07T15:15:05,856 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T15:15:05,857 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f8956274-ed33-4e16-82eb-d5d5911ec893' 2024-12-07T15:15:05,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T15:15:05,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f8956274-ed33-4e16-82eb-d5d5911ec893" 2024-12-07T15:15:05,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@210004e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:15:05,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6797ebe9c52,36915,-1] 2024-12-07T15:15:05,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T15:15:05,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:15:05,858 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53160, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T15:15:05,858 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18f50bff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:15:05,859 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35aa2c8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:15:05,859 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6797ebe9c52,36915,-1 for getting cluster id 2024-12-07T15:15:05,859 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T15:15:05,860 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f8956274-ed33-4e16-82eb-d5d5911ec893' 2024-12-07T15:15:05,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T15:15:05,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f8956274-ed33-4e16-82eb-d5d5911ec893" 2024-12-07T15:15:05,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2413dbc3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:15:05,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6797ebe9c52,36915,-1] 2024-12-07T15:15:05,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T15:15:05,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:15:05,861 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53186, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T15:15:05,861 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1598e780, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:15:05,862 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53f9edfd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:15:05,862 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6797ebe9c52,36915,-1 for getting cluster id 2024-12-07T15:15:05,862 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T15:15:05,862 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f8956274-ed33-4e16-82eb-d5d5911ec893' 2024-12-07T15:15:05,863 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T15:15:05,863 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f8956274-ed33-4e16-82eb-d5d5911ec893" 2024-12-07T15:15:05,863 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e58e80e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:15:05,863 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6797ebe9c52,36915,-1] 2024-12-07T15:15:05,863 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T15:15:05,863 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:15:05,864 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53208, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T15:15:05,864 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c778453, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:15:05,865 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42125f21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:15:05,865 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6797ebe9c52,36915,-1 for getting cluster id 2024-12-07T15:15:05,865 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T15:15:05,866 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f8956274-ed33-4e16-82eb-d5d5911ec893' 2024-12-07T15:15:05,866 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T15:15:05,866 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f8956274-ed33-4e16-82eb-d5d5911ec893" 2024-12-07T15:15:05,866 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73cb0a2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:15:05,866 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6797ebe9c52,36915,-1] 2024-12-07T15:15:05,866 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T15:15:05,866 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:15:05,867 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53220, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T15:15:05,867 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10e38d08, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:15:05,870 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64b126a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:15:05,870 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6797ebe9c52,36915,-1 for getting cluster id 2024-12-07T15:15:05,870 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T15:15:05,871 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f8956274-ed33-4e16-82eb-d5d5911ec893' 2024-12-07T15:15:05,871 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T15:15:05,871 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f8956274-ed33-4e16-82eb-d5d5911ec893" 2024-12-07T15:15:05,871 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@155a0378, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:15:05,871 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6797ebe9c52,36915,-1] 2024-12-07T15:15:05,871 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T15:15:05,871 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:15:05,872 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53238, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T15:15:05,872 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d232319, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:15:05,873 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fa71035, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:15:05,873 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6797ebe9c52,36915,-1 for getting cluster id 2024-12-07T15:15:05,873 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T15:15:05,873 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f8956274-ed33-4e16-82eb-d5d5911ec893' 2024-12-07T15:15:05,874 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T15:15:05,874 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f8956274-ed33-4e16-82eb-d5d5911ec893" 2024-12-07T15:15:05,874 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46e522a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:15:05,874 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6797ebe9c52,36915,-1] 2024-12-07T15:15:05,874 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T15:15:05,874 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:15:05,875 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53256, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T15:15:05,875 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f7fcbbe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:15:05,876 DEBUG [Thread-1588 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T15:15:05,876 DEBUG [Thread-1586 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T15:15:05,876 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T15:15:05,876 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6797ebe9c52,46009,1733584418401, seqNum=-1] 2024-12-07T15:15:05,876 DEBUG [Thread-1582 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T15:15:05,876 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6797ebe9c52,46009,1733584418401, seqNum=-1] 2024-12-07T15:15:05,877 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T15:15:05,877 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T15:15:05,877 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6797ebe9c52,46009,1733584418401, seqNum=-1] 2024-12-07T15:15:05,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-12-07T15:15:05,877 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T15:15:05,878 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42066, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T15:15:05,878 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42078, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T15:15:05,878 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T15:15:05,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-07T15:15:05,879 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42080, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T15:15:05,880 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5] 2024-12-07T15:15:05,880 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T15:15:05,880 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T15:15:05,880 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5] 2024-12-07T15:15:05,881 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5] 2024-12-07T15:15:05,885 DEBUG [Thread-1573 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T15:15:05,885 DEBUG [Thread-1580 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T15:15:05,886 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6797ebe9c52,46009,1733584418401, seqNum=-1] 2024-12-07T15:15:05,886 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T15:15:05,886 DEBUG [Thread-1571 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T15:15:05,887 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42088, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T15:15:05,887 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6797ebe9c52,46009,1733584418401, seqNum=-1] 2024-12-07T15:15:05,887 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6797ebe9c52,46009,1733584418401, seqNum=-1] 2024-12-07T15:15:05,888 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T15:15:05,888 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T15:15:05,888 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5] 2024-12-07T15:15:05,889 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42098, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T15:15:05,889 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42096, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T15:15:05,890 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5] 2024-12-07T15:15:05,891 DEBUG [Thread-1569 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T15:15:05,891 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6797ebe9c52,46009,1733584418401, seqNum=-1] 2024-12-07T15:15:05,892 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T15:15:05,892 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5] 2024-12-07T15:15:05,893 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42104, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T15:15:05,894 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5] 2024-12-07T15:15:05,901 DEBUG [Thread-1575 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T15:15:05,901 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6797ebe9c52,46009,1733584418401, seqNum=-1] 2024-12-07T15:15:05,902 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T15:15:05,905 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42120, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T15:15:05,908 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5] 2024-12-07T15:15:05,909 DEBUG [Thread-1577 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T15:15:05,910 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6797ebe9c52,46009,1733584418401, seqNum=-1] 2024-12-07T15:15:05,910 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T15:15:05,911 DEBUG [Thread-1584 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T15:15:05,912 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42136, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T15:15:05,913 DEBUG [Thread-1584 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6797ebe9c52,46009,1733584418401, seqNum=-1] 2024-12-07T15:15:05,914 DEBUG [Thread-1584 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T15:15:05,915 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42142, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T15:15:05,916 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c0d1e6963b45fdb2c1b5de83c5d92b34 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-07T15:15:05,916 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5] 2024-12-07T15:15:05,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0d1e6963b45fdb2c1b5de83c5d92b34, store=A 2024-12-07T15:15:05,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:15:05,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0d1e6963b45fdb2c1b5de83c5d92b34, store=B 2024-12-07T15:15:05,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:15:05,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0d1e6963b45fdb2c1b5de83c5d92b34, store=C 2024-12-07T15:15:05,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:15:05,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on c0d1e6963b45fdb2c1b5de83c5d92b34 2024-12-07T15:15:05,917 DEBUG [Thread-1584 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5] 2024-12-07T15:15:05,949 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207cae1fa44c0424f568ea68c336de6c18a_c0d1e6963b45fdb2c1b5de83c5d92b34 is 50, key is test_row_0/A:col10/1733584505910/Put/seqid=0 2024-12-07T15:15:05,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742171_1347 (size=12154) 2024-12-07T15:15:05,966 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:05,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42104 deadline: 1733584565953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:05,967 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:05,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42098 deadline: 1733584565951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:05,967 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:05,967 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:05,967 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:05,967 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:05,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42096 deadline: 1733584565954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:05,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:05,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:05,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:05,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:05,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42120 deadline: 1733584565956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:05,968 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733584505941, max timestamp=1733584505965Num uniques 859; 2024-12-07T15:15:05,969 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733584505941, max timestamp=1733584505965Num uniques 859; 2024-12-07T15:15:05,969 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-07T15:15:05,969 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-07T15:15:05,969 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733584505941, max timestamp=1733584505965Num uniques 859; 2024-12-07T15:15:05,969 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-07T15:15:05,969 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:05,969 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:05,969 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:05,970 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:05,970 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:05,970 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:05,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-07T15:15:06,033 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-07T15:15:06,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:06,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. as already flushing 2024-12-07T15:15:06,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:06,033 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:06,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:06,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:06,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:06,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42136 deadline: 1733584566064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:06,076 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:06,076 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:06,076 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:06,082 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:06,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42104 deadline: 1733584566073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:06,083 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:06,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:06,083 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:06,083 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:06,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42098 deadline: 1733584566073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:06,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:06,083 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:06,083 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:06,083 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:06,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42120 deadline: 1733584566073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:06,083 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:06,083 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:06,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:06,083 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:06,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42096 deadline: 1733584566073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:06,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:06,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:06,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:06,186 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-07T15:15:06,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:06,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. as already flushing 2024-12-07T15:15:06,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:06,187 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:06,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:06,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:06,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-07T15:15:06,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:06,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42136 deadline: 1733584566183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:06,194 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:06,194 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:06,194 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:06,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:06,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42104 deadline: 1733584566293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:06,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:06,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42098 deadline: 1733584566293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:06,301 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:06,301 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:06,301 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:06,302 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:06,302 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:06,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42120 deadline: 1733584566293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:06,302 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:06,302 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:06,302 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:06,302 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:06,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42096 deadline: 1733584566294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:06,302 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:06,302 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:06,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:06,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:06,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:06,340 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-07T15:15:06,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:06,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. as already flushing 2024-12-07T15:15:06,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:06,340 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:06,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:06,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:06,360 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:06,364 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207cae1fa44c0424f568ea68c336de6c18a_c0d1e6963b45fdb2c1b5de83c5d92b34 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207cae1fa44c0424f568ea68c336de6c18a_c0d1e6963b45fdb2c1b5de83c5d92b34 2024-12-07T15:15:06,365 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/A/57b9a640fdb54a228a375bbc92912932, store: [table=TestAcidGuarantees family=A region=c0d1e6963b45fdb2c1b5de83c5d92b34] 2024-12-07T15:15:06,365 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/A/57b9a640fdb54a228a375bbc92912932 is 175, key is test_row_0/A:col10/1733584505910/Put/seqid=0 2024-12-07T15:15:06,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742172_1348 (size=30955) 2024-12-07T15:15:06,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:06,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42136 deadline: 1733584566403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:06,416 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:06,417 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:06,417 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:06,492 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-07T15:15:06,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:06,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. as already flushing 2024-12-07T15:15:06,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:06,493 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:06,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:06,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:06,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-07T15:15:06,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:06,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42104 deadline: 1733584566613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:06,626 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:06,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42098 deadline: 1733584566613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:06,626 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:06,626 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:06,626 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:06,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:06,626 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:06,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:06,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:06,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42120 deadline: 1733584566613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:06,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:06,627 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:06,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42096 deadline: 1733584566613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:06,627 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:06,627 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:06,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:06,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:06,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:06,645 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-07T15:15:06,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:06,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. as already flushing 2024-12-07T15:15:06,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:06,646 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:06,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:06,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:06,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:06,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42136 deadline: 1733584566723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:06,731 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:06,732 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:06,732 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:06,770 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/A/57b9a640fdb54a228a375bbc92912932 2024-12-07T15:15:06,775 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-07T15:15:06,790 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/B/c40c8f4feb394d9f9251d3be339715f6 is 50, key is test_row_0/B:col10/1733584505910/Put/seqid=0 2024-12-07T15:15:06,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742173_1349 (size=12001) 2024-12-07T15:15:06,799 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-07T15:15:06,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:06,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. as already flushing 2024-12-07T15:15:06,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:06,799 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:06,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:06,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:06,952 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-07T15:15:06,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:06,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. as already flushing 2024-12-07T15:15:06,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:06,952 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:06,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:06,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:07,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-07T15:15:07,105 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-07T15:15:07,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:07,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. as already flushing 2024-12-07T15:15:07,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:07,105 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:07,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:07,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:07,140 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-07T15:15:07,142 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:07,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42098 deadline: 1733584567134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:07,142 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:07,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42104 deadline: 1733584567134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:07,142 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:07,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42096 deadline: 1733584567134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:07,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:07,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:07,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42120 deadline: 1733584567134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:07,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:07,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:07,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:07,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:07,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:07,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:07,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:07,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:07,143 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:07,143 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:07,144 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:07,194 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/B/c40c8f4feb394d9f9251d3be339715f6 2024-12-07T15:15:07,220 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/C/17cd0f78ce864d2cbe1bca9e6b9c7ee5 is 50, key is test_row_0/C:col10/1733584505910/Put/seqid=0 2024-12-07T15:15:07,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742174_1350 (size=12001) 2024-12-07T15:15:07,230 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/C/17cd0f78ce864d2cbe1bca9e6b9c7ee5 2024-12-07T15:15:07,237 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/A/57b9a640fdb54a228a375bbc92912932 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/57b9a640fdb54a228a375bbc92912932 2024-12-07T15:15:07,246 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/57b9a640fdb54a228a375bbc92912932, entries=150, sequenceid=15, filesize=30.2 K 2024-12-07T15:15:07,250 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/B/c40c8f4feb394d9f9251d3be339715f6 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/c40c8f4feb394d9f9251d3be339715f6 2024-12-07T15:15:07,249 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:07,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42136 deadline: 1733584567248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:07,250 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:07,250 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:07,250 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:07,254 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/c40c8f4feb394d9f9251d3be339715f6, entries=150, sequenceid=15, filesize=11.7 K 2024-12-07T15:15:07,254 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/C/17cd0f78ce864d2cbe1bca9e6b9c7ee5 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/17cd0f78ce864d2cbe1bca9e6b9c7ee5 2024-12-07T15:15:07,258 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-07T15:15:07,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:07,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. as already flushing 2024-12-07T15:15:07,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:07,259 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:07,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:07,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:07,260 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/17cd0f78ce864d2cbe1bca9e6b9c7ee5, entries=150, sequenceid=15, filesize=11.7 K 2024-12-07T15:15:07,268 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for c0d1e6963b45fdb2c1b5de83c5d92b34 in 1352ms, sequenceid=15, compaction requested=false 2024-12-07T15:15:07,269 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c0d1e6963b45fdb2c1b5de83c5d92b34: 2024-12-07T15:15:07,419 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-07T15:15:07,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:07,419 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing c0d1e6963b45fdb2c1b5de83c5d92b34 3/3 column families, dataSize=134.18 KB heapSize=370.92 KB 2024-12-07T15:15:07,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0d1e6963b45fdb2c1b5de83c5d92b34, store=A 2024-12-07T15:15:07,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T15:15:07,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0d1e6963b45fdb2c1b5de83c5d92b34, store=B 2024-12-07T15:15:07,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T15:15:07,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0d1e6963b45fdb2c1b5de83c5d92b34, store=C 2024-12-07T15:15:07,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T15:15:07,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412073a340af3c148469bb033779349b464b6_c0d1e6963b45fdb2c1b5de83c5d92b34 is 50, key is test_row_0/A:col10/1733584506032/Put/seqid=0 2024-12-07T15:15:07,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742175_1351 (size=12154) 2024-12-07T15:15:07,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:07,835 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412073a340af3c148469bb033779349b464b6_c0d1e6963b45fdb2c1b5de83c5d92b34 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412073a340af3c148469bb033779349b464b6_c0d1e6963b45fdb2c1b5de83c5d92b34 2024-12-07T15:15:07,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/A/3f0a391b29694c18ac1ec8470b9fff2e, store: [table=TestAcidGuarantees family=A region=c0d1e6963b45fdb2c1b5de83c5d92b34] 2024-12-07T15:15:07,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/A/3f0a391b29694c18ac1ec8470b9fff2e is 175, key is test_row_0/A:col10/1733584506032/Put/seqid=0 2024-12-07T15:15:07,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742176_1352 (size=30955) 2024-12-07T15:15:07,841 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=49, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/A/3f0a391b29694c18ac1ec8470b9fff2e 2024-12-07T15:15:07,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/B/0b4d44d95ed0416492b3e38b1366a5a7 is 50, key is test_row_0/B:col10/1733584506032/Put/seqid=0 2024-12-07T15:15:07,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742177_1353 (size=12001) 2024-12-07T15:15:08,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-07T15:15:08,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on c0d1e6963b45fdb2c1b5de83c5d92b34 2024-12-07T15:15:08,162 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. as already flushing 2024-12-07T15:15:08,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:08,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42096 deadline: 1733584568178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:08,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:08,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:08,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:08,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42120 deadline: 1733584568179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:08,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:08,194 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:08,194 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:08,194 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:08,206 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:08,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42104 deadline: 1733584568193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:08,206 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:08,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42098 deadline: 1733584568194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:08,206 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:08,206 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:08,206 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:08,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:08,207 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:08,207 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:08,253 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/B/0b4d44d95ed0416492b3e38b1366a5a7 2024-12-07T15:15:08,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:08,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42136 deadline: 1733584568253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:08,259 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to d6797ebe9c52,46009,1733584418401 for 'test_row_1' in c0d1e6963b45fdb2c1b5de83c5d92b34 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-07T15:15:08,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/C/571c54c593ef4f89a6e0ca6d1586d1af is 50, key is test_row_0/C:col10/1733584506032/Put/seqid=0 2024-12-07T15:15:08,260 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:08,260 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:08,261 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:08,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742178_1354 (size=12001) 2024-12-07T15:15:08,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:08,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42096 deadline: 1733584568303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:08,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:08,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42120 deadline: 1733584568303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:08,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:08,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:08,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:08,315 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:08,315 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:08,315 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:08,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:08,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42104 deadline: 1733584568313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:08,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:08,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:08,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:08,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42098 deadline: 1733584568313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:08,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:08,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:08,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:08,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:08,531 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:08,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42096 deadline: 1733584568524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:08,532 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:08,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:08,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:08,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:08,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42120 deadline: 1733584568524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:08,532 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:08,532 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:08,532 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:08,543 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:08,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42104 deadline: 1733584568533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:08,544 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:08,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42098 deadline: 1733584568533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:08,544 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:08,544 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:08,544 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:08,544 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:08,544 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:08,544 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:08,670 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/C/571c54c593ef4f89a6e0ca6d1586d1af 2024-12-07T15:15:08,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/A/3f0a391b29694c18ac1ec8470b9fff2e as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/3f0a391b29694c18ac1ec8470b9fff2e 2024-12-07T15:15:08,678 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/3f0a391b29694c18ac1ec8470b9fff2e, entries=150, sequenceid=49, filesize=30.2 K 2024-12-07T15:15:08,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/B/0b4d44d95ed0416492b3e38b1366a5a7 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/0b4d44d95ed0416492b3e38b1366a5a7 2024-12-07T15:15:08,682 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/0b4d44d95ed0416492b3e38b1366a5a7, entries=150, sequenceid=49, filesize=11.7 K 2024-12-07T15:15:08,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/C/571c54c593ef4f89a6e0ca6d1586d1af as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/571c54c593ef4f89a6e0ca6d1586d1af 2024-12-07T15:15:08,686 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/571c54c593ef4f89a6e0ca6d1586d1af, entries=150, sequenceid=49, filesize=11.7 K 2024-12-07T15:15:08,687 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~370.88 KB/379776, currentSize=53.67 KB/54960 for c0d1e6963b45fdb2c1b5de83c5d92b34 in 1268ms, sequenceid=49, compaction requested=false 2024-12-07T15:15:08,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for c0d1e6963b45fdb2c1b5de83c5d92b34: 2024-12-07T15:15:08,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:08,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-12-07T15:15:08,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-12-07T15:15:08,690 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-12-07T15:15:08,690 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8090 sec 2024-12-07T15:15:08,692 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 2.8150 sec 2024-12-07T15:15:08,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on c0d1e6963b45fdb2c1b5de83c5d92b34 2024-12-07T15:15:08,850 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c0d1e6963b45fdb2c1b5de83c5d92b34 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-07T15:15:08,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0d1e6963b45fdb2c1b5de83c5d92b34, store=A 2024-12-07T15:15:08,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:15:08,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0d1e6963b45fdb2c1b5de83c5d92b34, store=B 2024-12-07T15:15:08,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:15:08,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0d1e6963b45fdb2c1b5de83c5d92b34, store=C 2024-12-07T15:15:08,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:15:08,858 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207dee05949fae94e76851efe56f611f31e_c0d1e6963b45fdb2c1b5de83c5d92b34 is 50, key is test_row_0/A:col10/1733584508849/Put/seqid=0 2024-12-07T15:15:08,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742179_1355 (size=17034) 2024-12-07T15:15:08,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:08,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42096 deadline: 1733584568891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:08,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:08,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42120 deadline: 1733584568894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:08,905 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:08,905 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:08,905 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:08,905 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:08,905 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:08,905 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:08,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:08,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42104 deadline: 1733584568902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:08,908 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:08,908 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:08,908 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:08,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:08,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42098 deadline: 1733584568904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:08,917 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:08,917 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:08,917 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:09,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:09,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42096 deadline: 1733584569014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:09,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:09,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:09,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42120 deadline: 1733584569014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:09,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:09,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:09,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:09,024 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:09,024 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:09,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42104 deadline: 1733584569014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:09,024 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:09,024 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:09,024 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:09,024 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:09,030 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:09,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42098 deadline: 1733584569023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:09,030 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:09,030 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:09,030 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:09,243 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:09,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42096 deadline: 1733584569233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:09,243 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:09,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42120 deadline: 1733584569233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:09,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:09,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:09,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:09,243 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:09,243 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:09,243 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:09,243 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:09,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42104 deadline: 1733584569233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:09,244 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:09,244 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:09,244 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:09,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42098 deadline: 1733584569233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:09,244 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:09,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:09,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:09,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:09,274 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:09,278 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207dee05949fae94e76851efe56f611f31e_c0d1e6963b45fdb2c1b5de83c5d92b34 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207dee05949fae94e76851efe56f611f31e_c0d1e6963b45fdb2c1b5de83c5d92b34 2024-12-07T15:15:09,279 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/A/56887c1e10dd4ccf99a5e1ad17a00400, store: [table=TestAcidGuarantees family=A region=c0d1e6963b45fdb2c1b5de83c5d92b34] 2024-12-07T15:15:09,280 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/A/56887c1e10dd4ccf99a5e1ad17a00400 is 175, key is test_row_0/A:col10/1733584508849/Put/seqid=0 2024-12-07T15:15:09,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742180_1356 (size=48139) 2024-12-07T15:15:09,558 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:09,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42096 deadline: 1733584569553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:09,558 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:09,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:09,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:09,564 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:09,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42120 deadline: 1733584569553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:09,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:09,564 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:09,564 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:09,564 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:09,565 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:09,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42098 deadline: 1733584569554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:09,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42104 deadline: 1733584569554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:09,565 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:09,565 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:09,565 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:09,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:09,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:09,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:09,687 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=62, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/A/56887c1e10dd4ccf99a5e1ad17a00400 2024-12-07T15:15:09,694 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/B/7c1654d6354241ebae0bcadb2adcd274 is 50, key is test_row_0/B:col10/1733584508849/Put/seqid=0 2024-12-07T15:15:09,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742181_1357 (size=12001) 2024-12-07T15:15:10,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-07T15:15:10,044 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T15:15:10,044 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T15:15:10,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-12-07T15:15:10,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-07T15:15:10,046 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T15:15:10,046 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T15:15:10,047 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T15:15:10,082 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:10,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42096 deadline: 1733584570074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:10,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:10,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42120 deadline: 1733584570074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:10,083 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:10,083 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:10,083 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:10,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:10,083 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:10,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42098 deadline: 1733584570074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:10,083 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:10,083 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:10,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:10,084 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:10,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42104 deadline: 1733584570074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:10,084 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:10,084 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:10,084 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:10,084 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:10,084 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:10,110 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/B/7c1654d6354241ebae0bcadb2adcd274 2024-12-07T15:15:10,117 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/C/4ec29ff85c664d84b4f597a1416efb0e is 50, key is test_row_0/C:col10/1733584508849/Put/seqid=0 2024-12-07T15:15:10,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742182_1358 (size=12001) 2024-12-07T15:15:10,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-07T15:15:10,199 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-07T15:15:10,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:10,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. as already flushing 2024-12-07T15:15:10,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:10,199 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:10,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:10,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:10,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:10,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42136 deadline: 1733584570274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:10,285 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to d6797ebe9c52,46009,1733584418401 for 'test_row_1' in c0d1e6963b45fdb2c1b5de83c5d92b34 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4221 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-07T15:15:10,286 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:10,286 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:10,286 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:10,352 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-07T15:15:10,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:10,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. as already flushing 2024-12-07T15:15:10,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:10,352 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:10,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:10,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:10,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-07T15:15:10,505 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-07T15:15:10,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:10,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. as already flushing 2024-12-07T15:15:10,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:10,505 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:10,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:10,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:10,522 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/C/4ec29ff85c664d84b4f597a1416efb0e 2024-12-07T15:15:10,526 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/A/56887c1e10dd4ccf99a5e1ad17a00400 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/56887c1e10dd4ccf99a5e1ad17a00400 2024-12-07T15:15:10,530 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/56887c1e10dd4ccf99a5e1ad17a00400, entries=250, sequenceid=62, filesize=47.0 K 2024-12-07T15:15:10,531 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/B/7c1654d6354241ebae0bcadb2adcd274 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/7c1654d6354241ebae0bcadb2adcd274 2024-12-07T15:15:10,535 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/7c1654d6354241ebae0bcadb2adcd274, entries=150, sequenceid=62, filesize=11.7 K 2024-12-07T15:15:10,536 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/C/4ec29ff85c664d84b4f597a1416efb0e as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/4ec29ff85c664d84b4f597a1416efb0e 2024-12-07T15:15:10,540 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/4ec29ff85c664d84b4f597a1416efb0e, entries=150, sequenceid=62, filesize=11.7 K 2024-12-07T15:15:10,541 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for c0d1e6963b45fdb2c1b5de83c5d92b34 in 1691ms, sequenceid=62, compaction requested=true 2024-12-07T15:15:10,541 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c0d1e6963b45fdb2c1b5de83c5d92b34: 2024-12-07T15:15:10,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0d1e6963b45fdb2c1b5de83c5d92b34:A, priority=-2147483648, current under compaction store size is 1 2024-12-07T15:15:10,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:15:10,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0d1e6963b45fdb2c1b5de83c5d92b34:B, priority=-2147483648, current under compaction store size is 2 2024-12-07T15:15:10,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:15:10,541 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:15:10,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0d1e6963b45fdb2c1b5de83c5d92b34:C, priority=-2147483648, current under compaction store size is 3 2024-12-07T15:15:10,541 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:15:10,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T15:15:10,542 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:15:10,542 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110049 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:15:10,542 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1541): c0d1e6963b45fdb2c1b5de83c5d92b34/B is initiating minor compaction (all files) 2024-12-07T15:15:10,542 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1541): c0d1e6963b45fdb2c1b5de83c5d92b34/A is initiating minor compaction (all files) 2024-12-07T15:15:10,542 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0d1e6963b45fdb2c1b5de83c5d92b34/A in TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:10,542 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0d1e6963b45fdb2c1b5de83c5d92b34/B in TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:10,542 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/c40c8f4feb394d9f9251d3be339715f6, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/0b4d44d95ed0416492b3e38b1366a5a7, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/7c1654d6354241ebae0bcadb2adcd274] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp, totalSize=35.2 K 2024-12-07T15:15:10,542 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/57b9a640fdb54a228a375bbc92912932, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/3f0a391b29694c18ac1ec8470b9fff2e, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/56887c1e10dd4ccf99a5e1ad17a00400] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp, totalSize=107.5 K 2024-12-07T15:15:10,543 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:10,543 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. files: [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/57b9a640fdb54a228a375bbc92912932, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/3f0a391b29694c18ac1ec8470b9fff2e, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/56887c1e10dd4ccf99a5e1ad17a00400] 2024-12-07T15:15:10,543 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting c40c8f4feb394d9f9251d3be339715f6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733584505898 2024-12-07T15:15:10,543 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 57b9a640fdb54a228a375bbc92912932, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733584505898 2024-12-07T15:15:10,543 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b4d44d95ed0416492b3e38b1366a5a7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733584506032 2024-12-07T15:15:10,543 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f0a391b29694c18ac1ec8470b9fff2e, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733584506032 2024-12-07T15:15:10,543 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c1654d6354241ebae0bcadb2adcd274, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733584508191 2024-12-07T15:15:10,543 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 56887c1e10dd4ccf99a5e1ad17a00400, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733584508177 2024-12-07T15:15:10,553 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c0d1e6963b45fdb2c1b5de83c5d92b34] 2024-12-07T15:15:10,554 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0d1e6963b45fdb2c1b5de83c5d92b34#B#compaction#307 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:15:10,555 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412075b5935e189a248dc9c6536fd9da19f34_c0d1e6963b45fdb2c1b5de83c5d92b34 store=[table=TestAcidGuarantees family=A region=c0d1e6963b45fdb2c1b5de83c5d92b34] 2024-12-07T15:15:10,555 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/B/7b4a2eb32a644dc39ec4712a70002416 is 50, key is test_row_0/B:col10/1733584508849/Put/seqid=0 2024-12-07T15:15:10,556 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412075b5935e189a248dc9c6536fd9da19f34_c0d1e6963b45fdb2c1b5de83c5d92b34, store=[table=TestAcidGuarantees family=A region=c0d1e6963b45fdb2c1b5de83c5d92b34] 2024-12-07T15:15:10,557 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412075b5935e189a248dc9c6536fd9da19f34_c0d1e6963b45fdb2c1b5de83c5d92b34 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c0d1e6963b45fdb2c1b5de83c5d92b34] 2024-12-07T15:15:10,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742183_1359 (size=12104) 2024-12-07T15:15:10,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742184_1360 (size=4469) 2024-12-07T15:15:10,566 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/B/7b4a2eb32a644dc39ec4712a70002416 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/7b4a2eb32a644dc39ec4712a70002416 2024-12-07T15:15:10,571 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c0d1e6963b45fdb2c1b5de83c5d92b34/B of c0d1e6963b45fdb2c1b5de83c5d92b34 into 7b4a2eb32a644dc39ec4712a70002416(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:15:10,571 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0d1e6963b45fdb2c1b5de83c5d92b34: 2024-12-07T15:15:10,571 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., storeName=c0d1e6963b45fdb2c1b5de83c5d92b34/B, priority=13, startTime=1733584510541; duration=0sec 2024-12-07T15:15:10,571 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T15:15:10,571 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0d1e6963b45fdb2c1b5de83c5d92b34:B 2024-12-07T15:15:10,571 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:15:10,572 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:15:10,572 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1541): c0d1e6963b45fdb2c1b5de83c5d92b34/C is initiating minor compaction (all files) 2024-12-07T15:15:10,572 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0d1e6963b45fdb2c1b5de83c5d92b34/C in TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:10,572 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/17cd0f78ce864d2cbe1bca9e6b9c7ee5, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/571c54c593ef4f89a6e0ca6d1586d1af, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/4ec29ff85c664d84b4f597a1416efb0e] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp, totalSize=35.2 K 2024-12-07T15:15:10,572 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 17cd0f78ce864d2cbe1bca9e6b9c7ee5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733584505898 2024-12-07T15:15:10,573 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 571c54c593ef4f89a6e0ca6d1586d1af, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733584506032 2024-12-07T15:15:10,573 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ec29ff85c664d84b4f597a1416efb0e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733584508191 2024-12-07T15:15:10,579 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0d1e6963b45fdb2c1b5de83c5d92b34#C#compaction#309 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:15:10,580 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/C/02d92415e45b4646ba7549f05930f417 is 50, key is test_row_0/C:col10/1733584508849/Put/seqid=0 2024-12-07T15:15:10,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742185_1361 (size=12104) 2024-12-07T15:15:10,588 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/C/02d92415e45b4646ba7549f05930f417 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/02d92415e45b4646ba7549f05930f417 2024-12-07T15:15:10,592 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c0d1e6963b45fdb2c1b5de83c5d92b34/C of c0d1e6963b45fdb2c1b5de83c5d92b34 into 02d92415e45b4646ba7549f05930f417(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:15:10,592 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0d1e6963b45fdb2c1b5de83c5d92b34: 2024-12-07T15:15:10,592 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., storeName=c0d1e6963b45fdb2c1b5de83c5d92b34/C, priority=13, startTime=1733584510541; duration=0sec 2024-12-07T15:15:10,592 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:15:10,592 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0d1e6963b45fdb2c1b5de83c5d92b34:C 2024-12-07T15:15:10,657 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-07T15:15:10,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:10,658 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing c0d1e6963b45fdb2c1b5de83c5d92b34 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-07T15:15:10,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0d1e6963b45fdb2c1b5de83c5d92b34, store=A 2024-12-07T15:15:10,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:15:10,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0d1e6963b45fdb2c1b5de83c5d92b34, store=B 2024-12-07T15:15:10,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:15:10,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0d1e6963b45fdb2c1b5de83c5d92b34, store=C 2024-12-07T15:15:10,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:15:10,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207d27f85cb553441139e48d12df94f7634_c0d1e6963b45fdb2c1b5de83c5d92b34 is 50, key is test_row_0/A:col10/1733584508892/Put/seqid=0 2024-12-07T15:15:10,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742186_1362 (size=12154) 2024-12-07T15:15:10,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-07T15:15:10,964 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0d1e6963b45fdb2c1b5de83c5d92b34#A#compaction#308 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:15:10,964 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/A/962881e210e94b8ea9e71c291cb182f1 is 175, key is test_row_0/A:col10/1733584508849/Put/seqid=0 2024-12-07T15:15:10,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742187_1363 (size=31058) 2024-12-07T15:15:11,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:11,074 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207d27f85cb553441139e48d12df94f7634_c0d1e6963b45fdb2c1b5de83c5d92b34 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207d27f85cb553441139e48d12df94f7634_c0d1e6963b45fdb2c1b5de83c5d92b34 2024-12-07T15:15:11,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/A/5e952a52845d48b983ef9dcedd0e1430, store: [table=TestAcidGuarantees family=A region=c0d1e6963b45fdb2c1b5de83c5d92b34] 2024-12-07T15:15:11,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/A/5e952a52845d48b983ef9dcedd0e1430 is 175, key is test_row_0/A:col10/1733584508892/Put/seqid=0 2024-12-07T15:15:11,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742188_1364 (size=30955) 2024-12-07T15:15:11,088 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=83, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/A/5e952a52845d48b983ef9dcedd0e1430 2024-12-07T15:15:11,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on c0d1e6963b45fdb2c1b5de83c5d92b34 2024-12-07T15:15:11,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/B/1df34a03f1994f9fb90408fdf78c5360 is 50, key is test_row_0/B:col10/1733584508892/Put/seqid=0 2024-12-07T15:15:11,096 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. as already flushing 2024-12-07T15:15:11,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742189_1365 (size=12001) 2024-12-07T15:15:11,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:11,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:11,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42098 deadline: 1733584571112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:11,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42096 deadline: 1733584571114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:11,124 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:11,124 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:11,125 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:11,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:11,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:11,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:11,132 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:11,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42104 deadline: 1733584571123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:11,132 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:11,132 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:11,133 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:11,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42120 deadline: 1733584571124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:11,133 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:11,133 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:11,133 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:11,133 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:11,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-07T15:15:11,243 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:11,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42096 deadline: 1733584571233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:11,243 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:11,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:11,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42098 deadline: 1733584571233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:11,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:11,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:11,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:11,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:11,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:11,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:11,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42104 deadline: 1733584571243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:11,250 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:11,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:11,251 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:11,251 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:11,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42120 deadline: 1733584571243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:11,251 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:11,251 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:11,251 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:11,376 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/A/962881e210e94b8ea9e71c291cb182f1 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/962881e210e94b8ea9e71c291cb182f1 2024-12-07T15:15:11,380 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c0d1e6963b45fdb2c1b5de83c5d92b34/A of c0d1e6963b45fdb2c1b5de83c5d92b34 into 962881e210e94b8ea9e71c291cb182f1(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:15:11,380 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0d1e6963b45fdb2c1b5de83c5d92b34: 2024-12-07T15:15:11,380 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., storeName=c0d1e6963b45fdb2c1b5de83c5d92b34/A, priority=13, startTime=1733584510541; duration=0sec 2024-12-07T15:15:11,380 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:15:11,380 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0d1e6963b45fdb2c1b5de83c5d92b34:A 2024-12-07T15:15:11,461 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:11,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42096 deadline: 1733584571453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:11,461 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:11,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:11,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:11,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42098 deadline: 1733584571454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:11,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:11,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:11,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:11,461 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:11,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:11,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42104 deadline: 1733584571454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:11,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:11,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:11,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:11,473 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:11,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42120 deadline: 1733584571463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:11,474 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:11,474 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:11,474 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:11,505 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/B/1df34a03f1994f9fb90408fdf78c5360 2024-12-07T15:15:11,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/C/45b3169662524ce9a372c15c3b9f2895 is 50, key is test_row_0/C:col10/1733584508892/Put/seqid=0 2024-12-07T15:15:11,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742190_1366 (size=12001) 2024-12-07T15:15:11,517 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/C/45b3169662524ce9a372c15c3b9f2895 2024-12-07T15:15:11,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/A/5e952a52845d48b983ef9dcedd0e1430 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/5e952a52845d48b983ef9dcedd0e1430 2024-12-07T15:15:11,524 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/5e952a52845d48b983ef9dcedd0e1430, entries=150, sequenceid=83, filesize=30.2 K 2024-12-07T15:15:11,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/B/1df34a03f1994f9fb90408fdf78c5360 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/1df34a03f1994f9fb90408fdf78c5360 2024-12-07T15:15:11,528 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/1df34a03f1994f9fb90408fdf78c5360, entries=150, sequenceid=83, filesize=11.7 K 2024-12-07T15:15:11,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/C/45b3169662524ce9a372c15c3b9f2895 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/45b3169662524ce9a372c15c3b9f2895 2024-12-07T15:15:11,533 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/45b3169662524ce9a372c15c3b9f2895, entries=150, sequenceid=83, filesize=11.7 K 2024-12-07T15:15:11,534 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for c0d1e6963b45fdb2c1b5de83c5d92b34 in 876ms, sequenceid=83, compaction requested=false 2024-12-07T15:15:11,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for c0d1e6963b45fdb2c1b5de83c5d92b34: 2024-12-07T15:15:11,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:11,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-12-07T15:15:11,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-12-07T15:15:11,538 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-12-07T15:15:11,538 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4890 sec 2024-12-07T15:15:11,539 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 1.4940 sec 2024-12-07T15:15:11,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on c0d1e6963b45fdb2c1b5de83c5d92b34 2024-12-07T15:15:11,767 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c0d1e6963b45fdb2c1b5de83c5d92b34 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-07T15:15:11,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0d1e6963b45fdb2c1b5de83c5d92b34, store=A 2024-12-07T15:15:11,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:15:11,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0d1e6963b45fdb2c1b5de83c5d92b34, store=B 2024-12-07T15:15:11,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:15:11,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0d1e6963b45fdb2c1b5de83c5d92b34, store=C 2024-12-07T15:15:11,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:15:11,774 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412079cf547757d3347ed9e9423396b7b8082_c0d1e6963b45fdb2c1b5de83c5d92b34 is 50, key is test_row_0/A:col10/1733584511122/Put/seqid=0 2024-12-07T15:15:11,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742191_1367 (size=14594) 2024-12-07T15:15:11,820 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:11,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42120 deadline: 1733584571808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:11,820 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:11,820 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:11,820 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:11,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:11,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42096 deadline: 1733584571810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:11,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:11,827 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:11,827 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:11,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42104 deadline: 1733584571812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:11,827 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:11,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:11,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:11,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:11,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:11,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42098 deadline: 1733584571820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:11,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:11,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:11,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:11,932 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:11,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42120 deadline: 1733584571923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:11,933 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:11,933 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:11,933 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:11,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:11,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42096 deadline: 1733584571933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:11,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:11,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42104 deadline: 1733584571933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:11,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:11,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:11,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:11,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:11,944 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:11,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42098 deadline: 1733584571934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:11,944 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:11,944 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:11,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:11,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:11,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:12,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:12,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42120 deadline: 1733584572143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:12,151 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:12,151 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:12,151 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:12,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:12,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42096 deadline: 1733584572153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:12,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:12,165 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:12,165 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:12,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42104 deadline: 1733584572153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:12,165 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:12,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:12,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:12,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:12,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:12,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42098 deadline: 1733584572153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:12,166 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:12,166 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:12,166 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:12,181 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:12,185 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412079cf547757d3347ed9e9423396b7b8082_c0d1e6963b45fdb2c1b5de83c5d92b34 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412079cf547757d3347ed9e9423396b7b8082_c0d1e6963b45fdb2c1b5de83c5d92b34 2024-12-07T15:15:12,185 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/A/f99fc09bda1c45d4a4ac6d0d5aa4fc18, store: [table=TestAcidGuarantees family=A region=c0d1e6963b45fdb2c1b5de83c5d92b34] 2024-12-07T15:15:12,186 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/A/f99fc09bda1c45d4a4ac6d0d5aa4fc18 is 175, key is test_row_0/A:col10/1733584511122/Put/seqid=0 2024-12-07T15:15:12,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742192_1368 (size=39549) 2024-12-07T15:15:12,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-07T15:15:12,193 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T15:15:12,194 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T15:15:12,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-12-07T15:15:12,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-07T15:15:12,195 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T15:15:12,196 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T15:15:12,196 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T15:15:12,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-07T15:15:12,348 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-07T15:15:12,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:12,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. as already flushing 2024-12-07T15:15:12,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:12,348 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:12,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:12,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:12,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:12,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42120 deadline: 1733584572463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:12,469 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:12,469 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:12,469 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:12,481 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:12,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42096 deadline: 1733584572473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:12,481 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:12,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:12,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:12,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42104 deadline: 1733584572473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:12,482 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:12,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:12,482 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:12,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42098 deadline: 1733584572473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:12,482 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:12,482 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:12,482 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:12,482 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:12,482 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:12,501 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-07T15:15:12,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:12,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. as already flushing 2024-12-07T15:15:12,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:12,501 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:12,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:12,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:12,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-07T15:15:12,593 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=98, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/A/f99fc09bda1c45d4a4ac6d0d5aa4fc18 2024-12-07T15:15:12,604 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/B/348bb31c6b224c68bf1ee39bfa3c68f7 is 50, key is test_row_0/B:col10/1733584511122/Put/seqid=0 2024-12-07T15:15:12,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742193_1369 (size=12001) 2024-12-07T15:15:12,610 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/B/348bb31c6b224c68bf1ee39bfa3c68f7 2024-12-07T15:15:12,616 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/C/94d9e76399e84a6c92746c2385aeda3c is 50, key is test_row_0/C:col10/1733584511122/Put/seqid=0 2024-12-07T15:15:12,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742194_1370 (size=12001) 2024-12-07T15:15:12,654 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-07T15:15:12,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:12,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. as already flushing 2024-12-07T15:15:12,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:12,654 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:12,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:12,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:12,807 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-07T15:15:12,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:12,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. as already flushing 2024-12-07T15:15:12,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:12,808 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:12,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:12,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:12,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-07T15:15:12,960 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-07T15:15:12,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:12,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. as already flushing 2024-12-07T15:15:12,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:12,960 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:12,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:12,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:12,988 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:12,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42120 deadline: 1733584572973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:12,988 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:12,988 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:12,988 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:13,003 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:13,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42096 deadline: 1733584572993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:13,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:13,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:13,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42104 deadline: 1733584572993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:13,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:13,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:13,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:13,004 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:13,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42098 deadline: 1733584572993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:13,004 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:13,004 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:13,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:13,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:13,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:13,021 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/C/94d9e76399e84a6c92746c2385aeda3c 2024-12-07T15:15:13,025 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/A/f99fc09bda1c45d4a4ac6d0d5aa4fc18 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/f99fc09bda1c45d4a4ac6d0d5aa4fc18 2024-12-07T15:15:13,028 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/f99fc09bda1c45d4a4ac6d0d5aa4fc18, entries=200, sequenceid=98, filesize=38.6 K 2024-12-07T15:15:13,028 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/B/348bb31c6b224c68bf1ee39bfa3c68f7 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/348bb31c6b224c68bf1ee39bfa3c68f7 2024-12-07T15:15:13,032 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/348bb31c6b224c68bf1ee39bfa3c68f7, entries=150, sequenceid=98, filesize=11.7 K 2024-12-07T15:15:13,032 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/C/94d9e76399e84a6c92746c2385aeda3c as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/94d9e76399e84a6c92746c2385aeda3c 2024-12-07T15:15:13,036 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/94d9e76399e84a6c92746c2385aeda3c, entries=150, sequenceid=98, filesize=11.7 K 2024-12-07T15:15:13,036 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for c0d1e6963b45fdb2c1b5de83c5d92b34 in 1269ms, sequenceid=98, compaction requested=true 2024-12-07T15:15:13,036 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c0d1e6963b45fdb2c1b5de83c5d92b34: 2024-12-07T15:15:13,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0d1e6963b45fdb2c1b5de83c5d92b34:A, priority=-2147483648, current under compaction store size is 1 2024-12-07T15:15:13,037 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:15:13,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:15:13,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0d1e6963b45fdb2c1b5de83c5d92b34:B, priority=-2147483648, current under compaction store size is 2 2024-12-07T15:15:13,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:15:13,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0d1e6963b45fdb2c1b5de83c5d92b34:C, priority=-2147483648, current under compaction store size is 3 2024-12-07T15:15:13,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T15:15:13,037 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:15:13,037 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:15:13,037 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1541): c0d1e6963b45fdb2c1b5de83c5d92b34/A is initiating minor compaction (all files) 2024-12-07T15:15:13,038 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0d1e6963b45fdb2c1b5de83c5d92b34/A in TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:13,038 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/962881e210e94b8ea9e71c291cb182f1, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/5e952a52845d48b983ef9dcedd0e1430, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/f99fc09bda1c45d4a4ac6d0d5aa4fc18] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp, totalSize=99.2 K 2024-12-07T15:15:13,038 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:13,038 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. files: [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/962881e210e94b8ea9e71c291cb182f1, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/5e952a52845d48b983ef9dcedd0e1430, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/f99fc09bda1c45d4a4ac6d0d5aa4fc18] 2024-12-07T15:15:13,038 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:15:13,038 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1541): c0d1e6963b45fdb2c1b5de83c5d92b34/B is initiating minor compaction (all files) 2024-12-07T15:15:13,038 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0d1e6963b45fdb2c1b5de83c5d92b34/B in TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:13,038 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 962881e210e94b8ea9e71c291cb182f1, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733584508191 2024-12-07T15:15:13,038 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/7b4a2eb32a644dc39ec4712a70002416, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/1df34a03f1994f9fb90408fdf78c5360, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/348bb31c6b224c68bf1ee39bfa3c68f7] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp, totalSize=35.3 K 2024-12-07T15:15:13,038 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e952a52845d48b983ef9dcedd0e1430, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1733584508888 2024-12-07T15:15:13,038 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b4a2eb32a644dc39ec4712a70002416, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733584508191 2024-12-07T15:15:13,039 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting f99fc09bda1c45d4a4ac6d0d5aa4fc18, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733584511096 2024-12-07T15:15:13,039 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 1df34a03f1994f9fb90408fdf78c5360, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1733584508888 2024-12-07T15:15:13,039 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 348bb31c6b224c68bf1ee39bfa3c68f7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733584511121 2024-12-07T15:15:13,044 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c0d1e6963b45fdb2c1b5de83c5d92b34] 2024-12-07T15:15:13,050 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120749c5b6285bf84b1390fff85e4d36ca7e_c0d1e6963b45fdb2c1b5de83c5d92b34 store=[table=TestAcidGuarantees family=A region=c0d1e6963b45fdb2c1b5de83c5d92b34] 2024-12-07T15:15:13,050 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0d1e6963b45fdb2c1b5de83c5d92b34#B#compaction#317 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:15:13,051 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/B/f16cc676892645f7b5c94ed3a12a9b54 is 50, key is test_row_0/B:col10/1733584511122/Put/seqid=0 2024-12-07T15:15:13,052 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120749c5b6285bf84b1390fff85e4d36ca7e_c0d1e6963b45fdb2c1b5de83c5d92b34, store=[table=TestAcidGuarantees family=A region=c0d1e6963b45fdb2c1b5de83c5d92b34] 2024-12-07T15:15:13,052 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120749c5b6285bf84b1390fff85e4d36ca7e_c0d1e6963b45fdb2c1b5de83c5d92b34 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c0d1e6963b45fdb2c1b5de83c5d92b34] 2024-12-07T15:15:13,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742195_1371 (size=12207) 2024-12-07T15:15:13,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742196_1372 (size=4469) 2024-12-07T15:15:13,068 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0d1e6963b45fdb2c1b5de83c5d92b34#A#compaction#316 average throughput is 1.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:15:13,068 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/A/f4127c663eb842ebbcf63bfcc4dccd88 is 175, key is test_row_0/A:col10/1733584511122/Put/seqid=0 2024-12-07T15:15:13,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742197_1373 (size=31161) 2024-12-07T15:15:13,113 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-07T15:15:13,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:13,113 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing c0d1e6963b45fdb2c1b5de83c5d92b34 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-07T15:15:13,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0d1e6963b45fdb2c1b5de83c5d92b34, store=A 2024-12-07T15:15:13,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:15:13,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0d1e6963b45fdb2c1b5de83c5d92b34, store=B 2024-12-07T15:15:13,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:15:13,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0d1e6963b45fdb2c1b5de83c5d92b34, store=C 2024-12-07T15:15:13,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:15:13,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412072aaa8ea424654b78b402449604d2c4f0_c0d1e6963b45fdb2c1b5de83c5d92b34 is 50, key is test_row_0/A:col10/1733584511818/Put/seqid=0 2024-12-07T15:15:13,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742198_1374 (size=12154) 2024-12-07T15:15:13,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-07T15:15:13,462 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/B/f16cc676892645f7b5c94ed3a12a9b54 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/f16cc676892645f7b5c94ed3a12a9b54 2024-12-07T15:15:13,467 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c0d1e6963b45fdb2c1b5de83c5d92b34/B of c0d1e6963b45fdb2c1b5de83c5d92b34 into f16cc676892645f7b5c94ed3a12a9b54(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:15:13,467 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0d1e6963b45fdb2c1b5de83c5d92b34: 2024-12-07T15:15:13,467 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., storeName=c0d1e6963b45fdb2c1b5de83c5d92b34/B, priority=13, startTime=1733584513037; duration=0sec 2024-12-07T15:15:13,467 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T15:15:13,467 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0d1e6963b45fdb2c1b5de83c5d92b34:B 2024-12-07T15:15:13,467 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:15:13,468 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:15:13,468 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1541): c0d1e6963b45fdb2c1b5de83c5d92b34/C is initiating minor compaction (all files) 2024-12-07T15:15:13,468 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0d1e6963b45fdb2c1b5de83c5d92b34/C in TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:13,468 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/02d92415e45b4646ba7549f05930f417, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/45b3169662524ce9a372c15c3b9f2895, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/94d9e76399e84a6c92746c2385aeda3c] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp, totalSize=35.3 K 2024-12-07T15:15:13,468 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 02d92415e45b4646ba7549f05930f417, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733584508191 2024-12-07T15:15:13,469 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 45b3169662524ce9a372c15c3b9f2895, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1733584508888 2024-12-07T15:15:13,469 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 94d9e76399e84a6c92746c2385aeda3c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733584511121 2024-12-07T15:15:13,476 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0d1e6963b45fdb2c1b5de83c5d92b34#C#compaction#319 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:15:13,477 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/C/fe357de6d4a046079722a0f4f564a873 is 50, key is test_row_0/C:col10/1733584511122/Put/seqid=0 2024-12-07T15:15:13,482 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/A/f4127c663eb842ebbcf63bfcc4dccd88 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/f4127c663eb842ebbcf63bfcc4dccd88 2024-12-07T15:15:13,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742199_1375 (size=12207) 2024-12-07T15:15:13,487 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c0d1e6963b45fdb2c1b5de83c5d92b34/A of c0d1e6963b45fdb2c1b5de83c5d92b34 into f4127c663eb842ebbcf63bfcc4dccd88(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:15:13,487 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0d1e6963b45fdb2c1b5de83c5d92b34: 2024-12-07T15:15:13,488 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., storeName=c0d1e6963b45fdb2c1b5de83c5d92b34/A, priority=13, startTime=1733584513036; duration=0sec 2024-12-07T15:15:13,488 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:15:13,488 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0d1e6963b45fdb2c1b5de83c5d92b34:A 2024-12-07T15:15:13,489 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/C/fe357de6d4a046079722a0f4f564a873 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/fe357de6d4a046079722a0f4f564a873 2024-12-07T15:15:13,493 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c0d1e6963b45fdb2c1b5de83c5d92b34/C of c0d1e6963b45fdb2c1b5de83c5d92b34 into fe357de6d4a046079722a0f4f564a873(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:15:13,493 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0d1e6963b45fdb2c1b5de83c5d92b34: 2024-12-07T15:15:13,493 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., storeName=c0d1e6963b45fdb2c1b5de83c5d92b34/C, priority=13, startTime=1733584513037; duration=0sec 2024-12-07T15:15:13,493 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:15:13,493 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0d1e6963b45fdb2c1b5de83c5d92b34:C 2024-12-07T15:15:13,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:13,529 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412072aaa8ea424654b78b402449604d2c4f0_c0d1e6963b45fdb2c1b5de83c5d92b34 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412072aaa8ea424654b78b402449604d2c4f0_c0d1e6963b45fdb2c1b5de83c5d92b34 2024-12-07T15:15:13,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/A/37fff0c09a24417daffa9325aadfe2fa, store: [table=TestAcidGuarantees family=A region=c0d1e6963b45fdb2c1b5de83c5d92b34] 2024-12-07T15:15:13,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/A/37fff0c09a24417daffa9325aadfe2fa is 175, key is test_row_0/A:col10/1733584511818/Put/seqid=0 2024-12-07T15:15:13,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742200_1376 (size=30955) 2024-12-07T15:15:13,937 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=116, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/A/37fff0c09a24417daffa9325aadfe2fa 2024-12-07T15:15:13,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/B/48802220cc5c4d73a9df4bfbf8a907a8 is 50, key is test_row_0/B:col10/1733584511818/Put/seqid=0 2024-12-07T15:15:13,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742201_1377 (size=12001) 2024-12-07T15:15:13,950 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/B/48802220cc5c4d73a9df4bfbf8a907a8 2024-12-07T15:15:13,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/C/3ff67f62e5e74af29ae8730b07eb391f is 50, key is test_row_0/C:col10/1733584511818/Put/seqid=0 2024-12-07T15:15:13,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742202_1378 (size=12001) 2024-12-07T15:15:13,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on c0d1e6963b45fdb2c1b5de83c5d92b34 2024-12-07T15:15:13,999 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. as already flushing 2024-12-07T15:15:14,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:14,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42120 deadline: 1733584574024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:14,033 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:14,033 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:14,033 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:14,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:14,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42096 deadline: 1733584574029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:14,035 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:14,035 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:14,035 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:14,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:14,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42098 deadline: 1733584574031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:14,043 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:14,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:14,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:14,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:14,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42104 deadline: 1733584574032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:14,043 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:14,043 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:14,043 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:14,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:14,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42120 deadline: 1733584574143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:14,151 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:14,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42096 deadline: 1733584574143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:14,151 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:14,151 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:14,151 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:14,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:14,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:14,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:14,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:14,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42098 deadline: 1733584574153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:14,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:14,161 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:14,161 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:14,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42104 deadline: 1733584574153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:14,161 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:14,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:14,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:14,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:14,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:14,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42136 deadline: 1733584574313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:14,321 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to d6797ebe9c52,46009,1733584418401 for 'test_row_1' in c0d1e6963b45fdb2c1b5de83c5d92b34 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8257 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-07T15:15:14,322 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:14,322 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:14,322 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:14,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-07T15:15:14,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:14,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42096 deadline: 1733584574353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:14,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:14,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:14,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:14,364 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/C/3ff67f62e5e74af29ae8730b07eb391f 2024-12-07T15:15:14,368 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:14,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42120 deadline: 1733584574363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:14,368 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:14,368 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:14,368 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:14,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42098 deadline: 1733584574363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:14,368 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:14,368 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:14,369 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:14,369 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:14,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/A/37fff0c09a24417daffa9325aadfe2fa as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/37fff0c09a24417daffa9325aadfe2fa 2024-12-07T15:15:14,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42104 deadline: 1733584574363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:14,369 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:14,369 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:14,369 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:14,369 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:14,372 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/37fff0c09a24417daffa9325aadfe2fa, entries=150, sequenceid=116, filesize=30.2 K 2024-12-07T15:15:14,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/B/48802220cc5c4d73a9df4bfbf8a907a8 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/48802220cc5c4d73a9df4bfbf8a907a8 2024-12-07T15:15:14,376 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/48802220cc5c4d73a9df4bfbf8a907a8, entries=150, sequenceid=116, filesize=11.7 K 2024-12-07T15:15:14,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/C/3ff67f62e5e74af29ae8730b07eb391f as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/3ff67f62e5e74af29ae8730b07eb391f 2024-12-07T15:15:14,380 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/3ff67f62e5e74af29ae8730b07eb391f, entries=150, sequenceid=116, filesize=11.7 K 2024-12-07T15:15:14,380 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for c0d1e6963b45fdb2c1b5de83c5d92b34 in 1267ms, sequenceid=116, compaction requested=false 2024-12-07T15:15:14,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for c0d1e6963b45fdb2c1b5de83c5d92b34: 2024-12-07T15:15:14,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:14,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-12-07T15:15:14,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-12-07T15:15:14,383 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-12-07T15:15:14,384 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1860 sec 2024-12-07T15:15:14,385 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 2.1900 sec 2024-12-07T15:15:14,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on c0d1e6963b45fdb2c1b5de83c5d92b34 2024-12-07T15:15:14,666 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c0d1e6963b45fdb2c1b5de83c5d92b34 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-07T15:15:14,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0d1e6963b45fdb2c1b5de83c5d92b34, store=A 2024-12-07T15:15:14,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:15:14,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0d1e6963b45fdb2c1b5de83c5d92b34, store=B 2024-12-07T15:15:14,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:15:14,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0d1e6963b45fdb2c1b5de83c5d92b34, store=C 2024-12-07T15:15:14,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:15:14,673 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207edcaf453b13d4964bada512e85b2da37_c0d1e6963b45fdb2c1b5de83c5d92b34 is 50, key is test_row_0/A:col10/1733584513998/Put/seqid=0 2024-12-07T15:15:14,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742203_1379 (size=14744) 2024-12-07T15:15:14,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:14,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42098 deadline: 1733584574705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:14,709 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:14,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:14,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:14,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:14,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42120 deadline: 1733584574706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:14,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:14,719 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:14,719 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:14,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42104 deadline: 1733584574707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:14,719 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:14,719 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:14,719 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:14,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:14,719 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:14,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42096 deadline: 1733584574709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:14,720 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:14,720 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:14,720 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:14,823 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:14,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42098 deadline: 1733584574813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:14,824 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:14,824 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:14,824 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:14,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:14,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42120 deadline: 1733584574823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:14,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:14,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42104 deadline: 1733584574823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:14,834 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:14,834 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:14,834 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:14,834 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:14,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42096 deadline: 1733584574823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:14,834 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:14,834 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:14,834 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:14,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:14,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:14,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:15,043 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:15,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42098 deadline: 1733584575033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:15,044 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:15,044 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:15,044 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:15,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:15,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42120 deadline: 1733584575043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:15,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:15,054 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:15,054 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:15,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42104 deadline: 1733584575043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:15,054 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:15,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:15,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:15,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:15,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42096 deadline: 1733584575043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:15,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:15,055 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:15,055 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:15,055 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:15,077 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:15,081 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207edcaf453b13d4964bada512e85b2da37_c0d1e6963b45fdb2c1b5de83c5d92b34 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207edcaf453b13d4964bada512e85b2da37_c0d1e6963b45fdb2c1b5de83c5d92b34 2024-12-07T15:15:15,081 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/A/09815ac5f7e94af5bbcb9cd2d994d3b8, store: [table=TestAcidGuarantees family=A region=c0d1e6963b45fdb2c1b5de83c5d92b34] 2024-12-07T15:15:15,082 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/A/09815ac5f7e94af5bbcb9cd2d994d3b8 is 175, key is test_row_0/A:col10/1733584513998/Put/seqid=0 2024-12-07T15:15:15,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742204_1380 (size=39699) 2024-12-07T15:15:15,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:15,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42098 deadline: 1733584575353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:15,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:15,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:15,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:15,370 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:15,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42120 deadline: 1733584575363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:15,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:15,371 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:15,371 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:15,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42104 deadline: 1733584575363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:15,371 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:15,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:15,371 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:15,371 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:15,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42096 deadline: 1733584575363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:15,371 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:15,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:15,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:15,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:15,486 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=134, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/A/09815ac5f7e94af5bbcb9cd2d994d3b8 2024-12-07T15:15:15,493 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/B/7283bab41cc241939ecc7356499db1f5 is 50, key is test_row_0/B:col10/1733584513998/Put/seqid=0 2024-12-07T15:15:15,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742205_1381 (size=12101) 2024-12-07T15:15:15,869 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:15,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42098 deadline: 1733584575863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:15,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:15,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:15,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:15,886 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:15,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42120 deadline: 1733584575883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:15,886 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:15,886 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:15,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42104 deadline: 1733584575883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:15,886 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:15,886 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:15,888 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:15,888 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:15,888 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:15,895 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:15,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42096 deadline: 1733584575883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:15,896 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:15,896 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:15,896 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:15,900 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/B/7283bab41cc241939ecc7356499db1f5 2024-12-07T15:15:15,906 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/C/a5ec1a1f1c4844f6bff28968204b23b0 is 50, key is test_row_0/C:col10/1733584513998/Put/seqid=0 2024-12-07T15:15:15,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742206_1382 (size=12101) 2024-12-07T15:15:16,312 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/C/a5ec1a1f1c4844f6bff28968204b23b0 2024-12-07T15:15:16,320 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/A/09815ac5f7e94af5bbcb9cd2d994d3b8 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/09815ac5f7e94af5bbcb9cd2d994d3b8 2024-12-07T15:15:16,326 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/09815ac5f7e94af5bbcb9cd2d994d3b8, entries=200, sequenceid=134, filesize=38.8 K 2024-12-07T15:15:16,327 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/B/7283bab41cc241939ecc7356499db1f5 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/7283bab41cc241939ecc7356499db1f5 2024-12-07T15:15:16,331 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/7283bab41cc241939ecc7356499db1f5, entries=150, sequenceid=134, filesize=11.8 K 2024-12-07T15:15:16,332 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/C/a5ec1a1f1c4844f6bff28968204b23b0 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/a5ec1a1f1c4844f6bff28968204b23b0 2024-12-07T15:15:16,340 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/a5ec1a1f1c4844f6bff28968204b23b0, entries=150, sequenceid=134, filesize=11.8 K 2024-12-07T15:15:16,341 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for c0d1e6963b45fdb2c1b5de83c5d92b34 in 1675ms, sequenceid=134, compaction requested=true 2024-12-07T15:15:16,341 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c0d1e6963b45fdb2c1b5de83c5d92b34: 2024-12-07T15:15:16,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0d1e6963b45fdb2c1b5de83c5d92b34:A, priority=-2147483648, current under compaction store size is 1 2024-12-07T15:15:16,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:15:16,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0d1e6963b45fdb2c1b5de83c5d92b34:B, priority=-2147483648, current under compaction store size is 2 2024-12-07T15:15:16,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:15:16,342 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:15:16,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0d1e6963b45fdb2c1b5de83c5d92b34:C, priority=-2147483648, current under compaction store size is 3 2024-12-07T15:15:16,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T15:15:16,342 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:15:16,343 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:15:16,343 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1541): c0d1e6963b45fdb2c1b5de83c5d92b34/B is initiating minor compaction (all files) 2024-12-07T15:15:16,343 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0d1e6963b45fdb2c1b5de83c5d92b34/B in TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:16,343 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/f16cc676892645f7b5c94ed3a12a9b54, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/48802220cc5c4d73a9df4bfbf8a907a8, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/7283bab41cc241939ecc7356499db1f5] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp, totalSize=35.5 K 2024-12-07T15:15:16,343 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101815 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:15:16,343 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1541): c0d1e6963b45fdb2c1b5de83c5d92b34/A is initiating minor compaction (all files) 2024-12-07T15:15:16,343 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0d1e6963b45fdb2c1b5de83c5d92b34/A in TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:16,343 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/f4127c663eb842ebbcf63bfcc4dccd88, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/37fff0c09a24417daffa9325aadfe2fa, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/09815ac5f7e94af5bbcb9cd2d994d3b8] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp, totalSize=99.4 K 2024-12-07T15:15:16,343 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:16,343 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. files: [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/f4127c663eb842ebbcf63bfcc4dccd88, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/37fff0c09a24417daffa9325aadfe2fa, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/09815ac5f7e94af5bbcb9cd2d994d3b8] 2024-12-07T15:15:16,344 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting f16cc676892645f7b5c94ed3a12a9b54, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733584511121 2024-12-07T15:15:16,344 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting f4127c663eb842ebbcf63bfcc4dccd88, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733584511121 2024-12-07T15:15:16,344 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 48802220cc5c4d73a9df4bfbf8a907a8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1733584511809 2024-12-07T15:15:16,344 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 37fff0c09a24417daffa9325aadfe2fa, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1733584511809 2024-12-07T15:15:16,345 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 7283bab41cc241939ecc7356499db1f5, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1733584513998 2024-12-07T15:15:16,345 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 09815ac5f7e94af5bbcb9cd2d994d3b8, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1733584513998 2024-12-07T15:15:16,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-07T15:15:16,353 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T15:15:16,354 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T15:15:16,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-12-07T15:15:16,356 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T15:15:16,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-07T15:15:16,357 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T15:15:16,357 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T15:15:16,369 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c0d1e6963b45fdb2c1b5de83c5d92b34] 2024-12-07T15:15:16,383 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0d1e6963b45fdb2c1b5de83c5d92b34#B#compaction#326 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:15:16,383 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/B/3693612937ce43228c055bb19b9f9c05 is 50, key is test_row_0/B:col10/1733584513998/Put/seqid=0 2024-12-07T15:15:16,397 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241207742e2cff30934005ad676da023eb808f_c0d1e6963b45fdb2c1b5de83c5d92b34 store=[table=TestAcidGuarantees family=A region=c0d1e6963b45fdb2c1b5de83c5d92b34] 2024-12-07T15:15:16,399 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241207742e2cff30934005ad676da023eb808f_c0d1e6963b45fdb2c1b5de83c5d92b34, store=[table=TestAcidGuarantees family=A region=c0d1e6963b45fdb2c1b5de83c5d92b34] 2024-12-07T15:15:16,399 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207742e2cff30934005ad676da023eb808f_c0d1e6963b45fdb2c1b5de83c5d92b34 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c0d1e6963b45fdb2c1b5de83c5d92b34] 2024-12-07T15:15:16,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742207_1383 (size=12409) 2024-12-07T15:15:16,436 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/B/3693612937ce43228c055bb19b9f9c05 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/3693612937ce43228c055bb19b9f9c05 2024-12-07T15:15:16,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742208_1384 (size=4469) 2024-12-07T15:15:16,438 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0d1e6963b45fdb2c1b5de83c5d92b34#A#compaction#325 average throughput is 0.35 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:15:16,439 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/A/744219665d7741018b2945e6a9e5f1cf is 175, key is test_row_0/A:col10/1733584513998/Put/seqid=0 2024-12-07T15:15:16,462 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c0d1e6963b45fdb2c1b5de83c5d92b34/B of c0d1e6963b45fdb2c1b5de83c5d92b34 into 3693612937ce43228c055bb19b9f9c05(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:15:16,462 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0d1e6963b45fdb2c1b5de83c5d92b34: 2024-12-07T15:15:16,462 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., storeName=c0d1e6963b45fdb2c1b5de83c5d92b34/B, priority=13, startTime=1733584516342; duration=0sec 2024-12-07T15:15:16,462 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T15:15:16,462 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0d1e6963b45fdb2c1b5de83c5d92b34:B 2024-12-07T15:15:16,462 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:15:16,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-07T15:15:16,465 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:15:16,465 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1541): c0d1e6963b45fdb2c1b5de83c5d92b34/C is initiating minor compaction (all files) 2024-12-07T15:15:16,466 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0d1e6963b45fdb2c1b5de83c5d92b34/C in TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:16,466 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/fe357de6d4a046079722a0f4f564a873, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/3ff67f62e5e74af29ae8730b07eb391f, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/a5ec1a1f1c4844f6bff28968204b23b0] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp, totalSize=35.5 K 2024-12-07T15:15:16,466 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting fe357de6d4a046079722a0f4f564a873, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733584511121 2024-12-07T15:15:16,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742209_1385 (size=31363) 2024-12-07T15:15:16,467 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ff67f62e5e74af29ae8730b07eb391f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1733584511809 2024-12-07T15:15:16,467 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting a5ec1a1f1c4844f6bff28968204b23b0, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1733584513998 2024-12-07T15:15:16,473 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/A/744219665d7741018b2945e6a9e5f1cf as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/744219665d7741018b2945e6a9e5f1cf 2024-12-07T15:15:16,478 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c0d1e6963b45fdb2c1b5de83c5d92b34/A of c0d1e6963b45fdb2c1b5de83c5d92b34 into 744219665d7741018b2945e6a9e5f1cf(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:15:16,478 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0d1e6963b45fdb2c1b5de83c5d92b34: 2024-12-07T15:15:16,478 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., storeName=c0d1e6963b45fdb2c1b5de83c5d92b34/A, priority=13, startTime=1733584516341; duration=0sec 2024-12-07T15:15:16,478 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:15:16,478 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0d1e6963b45fdb2c1b5de83c5d92b34:A 2024-12-07T15:15:16,479 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0d1e6963b45fdb2c1b5de83c5d92b34#C#compaction#327 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:15:16,479 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/C/bd3e663d4f6c4ae1a228f1692245ca6f is 50, key is test_row_0/C:col10/1733584513998/Put/seqid=0 2024-12-07T15:15:16,509 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-07T15:15:16,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:16,510 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing c0d1e6963b45fdb2c1b5de83c5d92b34 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-07T15:15:16,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0d1e6963b45fdb2c1b5de83c5d92b34, store=A 2024-12-07T15:15:16,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:15:16,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0d1e6963b45fdb2c1b5de83c5d92b34, store=B 2024-12-07T15:15:16,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:15:16,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0d1e6963b45fdb2c1b5de83c5d92b34, store=C 2024-12-07T15:15:16,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:15:16,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742210_1386 (size=12409) 2024-12-07T15:15:16,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207eb8d1f5b6cc248ec895a685a008cc351_c0d1e6963b45fdb2c1b5de83c5d92b34 is 50, key is test_row_0/A:col10/1733584514703/Put/seqid=0 2024-12-07T15:15:16,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742211_1387 (size=12304) 2024-12-07T15:15:16,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:16,561 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207eb8d1f5b6cc248ec895a685a008cc351_c0d1e6963b45fdb2c1b5de83c5d92b34 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207eb8d1f5b6cc248ec895a685a008cc351_c0d1e6963b45fdb2c1b5de83c5d92b34 2024-12-07T15:15:16,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/A/a62ecbfb2b6b4dd9b9a9f839204f3280, store: [table=TestAcidGuarantees family=A region=c0d1e6963b45fdb2c1b5de83c5d92b34] 2024-12-07T15:15:16,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/A/a62ecbfb2b6b4dd9b9a9f839204f3280 is 175, key is test_row_0/A:col10/1733584514703/Put/seqid=0 2024-12-07T15:15:16,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742212_1388 (size=31105) 2024-12-07T15:15:16,605 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=153, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/A/a62ecbfb2b6b4dd9b9a9f839204f3280 2024-12-07T15:15:16,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/B/66bc69407eaf4d7ebf46f71c7701ebe9 is 50, key is test_row_0/B:col10/1733584514703/Put/seqid=0 2024-12-07T15:15:16,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742213_1389 (size=12151) 2024-12-07T15:15:16,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-07T15:15:16,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on c0d1e6963b45fdb2c1b5de83c5d92b34 2024-12-07T15:15:16,890 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. as already flushing 2024-12-07T15:15:16,923 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/C/bd3e663d4f6c4ae1a228f1692245ca6f as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/bd3e663d4f6c4ae1a228f1692245ca6f 2024-12-07T15:15:16,928 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c0d1e6963b45fdb2c1b5de83c5d92b34/C of c0d1e6963b45fdb2c1b5de83c5d92b34 into bd3e663d4f6c4ae1a228f1692245ca6f(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:15:16,928 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0d1e6963b45fdb2c1b5de83c5d92b34: 2024-12-07T15:15:16,928 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., storeName=c0d1e6963b45fdb2c1b5de83c5d92b34/C, priority=13, startTime=1733584516342; duration=0sec 2024-12-07T15:15:16,928 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:15:16,928 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0d1e6963b45fdb2c1b5de83c5d92b34:C 2024-12-07T15:15:16,930 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:16,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42120 deadline: 1733584576918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:16,931 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:16,931 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:16,931 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:16,938 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:16,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42104 deadline: 1733584576926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:16,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:16,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:16,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:16,941 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:16,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42096 deadline: 1733584576927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:16,941 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:16,941 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:16,941 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:16,942 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:16,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42098 deadline: 1733584576929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:16,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:16,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:16,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:16,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-07T15:15:17,042 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/B/66bc69407eaf4d7ebf46f71c7701ebe9 2024-12-07T15:15:17,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:17,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42120 deadline: 1733584577033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:17,046 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:17,046 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:17,046 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:17,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:17,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42104 deadline: 1733584577044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:17,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:17,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42096 deadline: 1733584577044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:17,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:17,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:17,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:17,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42098 deadline: 1733584577044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:17,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:17,049 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:17,049 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:17,049 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:17,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:17,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:17,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:17,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/C/1993c4695bf445c28ba2a5ee972c3989 is 50, key is test_row_0/C:col10/1733584514703/Put/seqid=0 2024-12-07T15:15:17,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742214_1390 (size=12151) 2024-12-07T15:15:17,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:17,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42120 deadline: 1733584577253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:17,260 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:17,260 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:17,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42096 deadline: 1733584577254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:17,260 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:17,260 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:17,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:17,260 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:17,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:17,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:17,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42098 deadline: 1733584577254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:17,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:17,261 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:17,261 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:17,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:17,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42104 deadline: 1733584577254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:17,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:17,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:17,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:17,458 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/C/1993c4695bf445c28ba2a5ee972c3989 2024-12-07T15:15:17,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/A/a62ecbfb2b6b4dd9b9a9f839204f3280 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/a62ecbfb2b6b4dd9b9a9f839204f3280 2024-12-07T15:15:17,466 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/a62ecbfb2b6b4dd9b9a9f839204f3280, entries=150, sequenceid=153, filesize=30.4 K 2024-12-07T15:15:17,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/B/66bc69407eaf4d7ebf46f71c7701ebe9 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/66bc69407eaf4d7ebf46f71c7701ebe9 2024-12-07T15:15:17,470 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/66bc69407eaf4d7ebf46f71c7701ebe9, entries=150, sequenceid=153, filesize=11.9 K 2024-12-07T15:15:17,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/C/1993c4695bf445c28ba2a5ee972c3989 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/1993c4695bf445c28ba2a5ee972c3989 2024-12-07T15:15:17,474 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/1993c4695bf445c28ba2a5ee972c3989, entries=150, sequenceid=153, filesize=11.9 K 2024-12-07T15:15:17,475 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for c0d1e6963b45fdb2c1b5de83c5d92b34 in 965ms, sequenceid=153, compaction requested=false 2024-12-07T15:15:17,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for c0d1e6963b45fdb2c1b5de83c5d92b34: 2024-12-07T15:15:17,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:17,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-12-07T15:15:17,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-12-07T15:15:17,479 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-12-07T15:15:17,479 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1200 sec 2024-12-07T15:15:17,480 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 1.1250 sec 2024-12-07T15:15:17,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-07T15:15:17,493 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T15:15:17,493 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T15:15:17,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-12-07T15:15:17,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-07T15:15:17,495 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T15:15:17,495 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T15:15:17,495 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T15:15:17,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on c0d1e6963b45fdb2c1b5de83c5d92b34 2024-12-07T15:15:17,569 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c0d1e6963b45fdb2c1b5de83c5d92b34 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-07T15:15:17,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0d1e6963b45fdb2c1b5de83c5d92b34, store=A 2024-12-07T15:15:17,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:15:17,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0d1e6963b45fdb2c1b5de83c5d92b34, store=B 2024-12-07T15:15:17,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:15:17,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0d1e6963b45fdb2c1b5de83c5d92b34, store=C 2024-12-07T15:15:17,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:15:17,576 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207cb98d10c0b3141b1b3e63fb0893f033d_c0d1e6963b45fdb2c1b5de83c5d92b34 is 50, key is test_row_0/A:col10/1733584516914/Put/seqid=0 2024-12-07T15:15:17,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742215_1391 (size=14794) 2024-12-07T15:15:17,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-07T15:15:17,608 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:17,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42104 deadline: 1733584577603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:17,609 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:17,609 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:17,609 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:17,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:17,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42120 deadline: 1733584577607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:17,614 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:17,614 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:17,614 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:17,618 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:17,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42096 deadline: 1733584577607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:17,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:17,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:17,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:17,620 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:17,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42098 deadline: 1733584577608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:17,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:17,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:17,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:17,646 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-07T15:15:17,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:17,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. as already flushing 2024-12-07T15:15:17,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:17,647 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:17,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:17,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:17,719 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:17,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42104 deadline: 1733584577713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:17,720 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:17,720 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:17,720 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:17,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:17,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42120 deadline: 1733584577723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:17,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:17,733 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:17,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42096 deadline: 1733584577723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:17,733 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:17,733 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:17,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:17,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:17,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:17,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42098 deadline: 1733584577724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:17,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:17,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:17,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:17,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:17,799 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-07T15:15:17,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:17,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. as already flushing 2024-12-07T15:15:17,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:17,799 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:17,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:17,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:17,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-07T15:15:17,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:17,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42104 deadline: 1733584577923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:17,935 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:17,935 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:17,935 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:17,951 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:17,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42120 deadline: 1733584577943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:17,951 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:17,951 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:17,952 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:17,952 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:17,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42096 deadline: 1733584577944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:17,952 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:17,952 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:17,952 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-07T15:15:17,952 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:17,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42098 deadline: 1733584577944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:17,952 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:17,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:17,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. as already flushing 2024-12-07T15:15:17,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:17,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:17,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:17,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:17,952 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:17,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:17,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:17,981 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:17,984 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207cb98d10c0b3141b1b3e63fb0893f033d_c0d1e6963b45fdb2c1b5de83c5d92b34 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207cb98d10c0b3141b1b3e63fb0893f033d_c0d1e6963b45fdb2c1b5de83c5d92b34 2024-12-07T15:15:17,985 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/A/2e21a1818f2f4126a834d2f4f70f64b2, store: [table=TestAcidGuarantees family=A region=c0d1e6963b45fdb2c1b5de83c5d92b34] 2024-12-07T15:15:17,985 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/A/2e21a1818f2f4126a834d2f4f70f64b2 is 175, key is test_row_0/A:col10/1733584516914/Put/seqid=0 2024-12-07T15:15:17,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742216_1392 (size=39749) 2024-12-07T15:15:18,105 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-07T15:15:18,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:18,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. as already flushing 2024-12-07T15:15:18,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:18,105 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:18,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:18,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:18,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-07T15:15:18,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:18,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42104 deadline: 1733584578243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:18,253 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:18,253 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:18,253 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:18,258 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-07T15:15:18,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:18,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. as already flushing 2024-12-07T15:15:18,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:18,258 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:18,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:18,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:18,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:18,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42120 deadline: 1733584578263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:18,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:18,273 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:18,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42096 deadline: 1733584578263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:18,273 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:18,273 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:18,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:18,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:18,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:18,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42098 deadline: 1733584578263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:18,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:18,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:18,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:18,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:18,390 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=170, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/A/2e21a1818f2f4126a834d2f4f70f64b2 2024-12-07T15:15:18,396 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/B/c888812db65e4ee6b5650ec07a4d4ef2 is 50, key is test_row_0/B:col10/1733584516914/Put/seqid=0 2024-12-07T15:15:18,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742217_1393 (size=12151) 2024-12-07T15:15:18,411 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-07T15:15:18,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:18,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. as already flushing 2024-12-07T15:15:18,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:18,411 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:18,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:18,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:18,564 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-07T15:15:18,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:18,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. as already flushing 2024-12-07T15:15:18,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:18,564 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:18,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:18,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:18,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-07T15:15:18,717 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-07T15:15:18,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:18,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. as already flushing 2024-12-07T15:15:18,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:18,717 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:18,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:18,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:18,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:18,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42104 deadline: 1733584578763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:18,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:18,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:18,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:18,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:18,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42120 deadline: 1733584578783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:18,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:18,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42096 deadline: 1733584578783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:18,788 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:18,788 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:18,788 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:18,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:18,788 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:18,788 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:18,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42098 deadline: 1733584578783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:18,788 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:18,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:18,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:18,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:18,800 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/B/c888812db65e4ee6b5650ec07a4d4ef2 2024-12-07T15:15:18,807 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/C/7f8cde578a1b471cbc15a1ea4e2a51c8 is 50, key is test_row_0/C:col10/1733584516914/Put/seqid=0 2024-12-07T15:15:18,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742218_1394 (size=12151) 2024-12-07T15:15:18,869 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-07T15:15:18,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:18,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. as already flushing 2024-12-07T15:15:18,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:18,870 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:18,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:18,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:19,023 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-07T15:15:19,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:19,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. as already flushing 2024-12-07T15:15:19,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:19,023 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:19,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:19,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:19,176 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-07T15:15:19,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:19,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. as already flushing 2024-12-07T15:15:19,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:19,176 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:19,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:19,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:19,211 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/C/7f8cde578a1b471cbc15a1ea4e2a51c8 2024-12-07T15:15:19,216 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/A/2e21a1818f2f4126a834d2f4f70f64b2 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/2e21a1818f2f4126a834d2f4f70f64b2 2024-12-07T15:15:19,219 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/2e21a1818f2f4126a834d2f4f70f64b2, entries=200, sequenceid=170, filesize=38.8 K 2024-12-07T15:15:19,220 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/B/c888812db65e4ee6b5650ec07a4d4ef2 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/c888812db65e4ee6b5650ec07a4d4ef2 2024-12-07T15:15:19,223 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/c888812db65e4ee6b5650ec07a4d4ef2, entries=150, sequenceid=170, filesize=11.9 K 2024-12-07T15:15:19,223 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/C/7f8cde578a1b471cbc15a1ea4e2a51c8 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/7f8cde578a1b471cbc15a1ea4e2a51c8 2024-12-07T15:15:19,226 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/7f8cde578a1b471cbc15a1ea4e2a51c8, entries=150, sequenceid=170, filesize=11.9 K 2024-12-07T15:15:19,227 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for c0d1e6963b45fdb2c1b5de83c5d92b34 in 1658ms, sequenceid=170, compaction requested=true 2024-12-07T15:15:19,227 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c0d1e6963b45fdb2c1b5de83c5d92b34: 2024-12-07T15:15:19,227 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:15:19,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0d1e6963b45fdb2c1b5de83c5d92b34:A, priority=-2147483648, current under compaction store size is 1 2024-12-07T15:15:19,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:15:19,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0d1e6963b45fdb2c1b5de83c5d92b34:B, priority=-2147483648, current under compaction store size is 2 2024-12-07T15:15:19,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:15:19,228 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:15:19,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0d1e6963b45fdb2c1b5de83c5d92b34:C, priority=-2147483648, current under compaction store size is 3 2024-12-07T15:15:19,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T15:15:19,233 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102217 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:15:19,233 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1541): c0d1e6963b45fdb2c1b5de83c5d92b34/A is initiating minor compaction (all files) 2024-12-07T15:15:19,233 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0d1e6963b45fdb2c1b5de83c5d92b34/A in TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:19,233 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:15:19,233 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/744219665d7741018b2945e6a9e5f1cf, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/a62ecbfb2b6b4dd9b9a9f839204f3280, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/2e21a1818f2f4126a834d2f4f70f64b2] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp, totalSize=99.8 K 2024-12-07T15:15:19,233 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1541): c0d1e6963b45fdb2c1b5de83c5d92b34/B is initiating minor compaction (all files) 2024-12-07T15:15:19,233 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:19,233 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0d1e6963b45fdb2c1b5de83c5d92b34/B in TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:19,233 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. files: [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/744219665d7741018b2945e6a9e5f1cf, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/a62ecbfb2b6b4dd9b9a9f839204f3280, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/2e21a1818f2f4126a834d2f4f70f64b2] 2024-12-07T15:15:19,233 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/3693612937ce43228c055bb19b9f9c05, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/66bc69407eaf4d7ebf46f71c7701ebe9, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/c888812db65e4ee6b5650ec07a4d4ef2] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp, totalSize=35.9 K 2024-12-07T15:15:19,233 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 744219665d7741018b2945e6a9e5f1cf, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1733584513998 2024-12-07T15:15:19,233 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 3693612937ce43228c055bb19b9f9c05, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1733584513998 2024-12-07T15:15:19,234 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting a62ecbfb2b6b4dd9b9a9f839204f3280, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1733584514703 2024-12-07T15:15:19,234 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 66bc69407eaf4d7ebf46f71c7701ebe9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1733584514703 2024-12-07T15:15:19,234 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting c888812db65e4ee6b5650ec07a4d4ef2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733584516914 2024-12-07T15:15:19,234 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e21a1818f2f4126a834d2f4f70f64b2, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733584516914 2024-12-07T15:15:19,241 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c0d1e6963b45fdb2c1b5de83c5d92b34] 2024-12-07T15:15:19,241 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0d1e6963b45fdb2c1b5de83c5d92b34#B#compaction#334 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:15:19,242 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/B/8e0affbbc9cd436eae73d5a2d3b6ef98 is 50, key is test_row_0/B:col10/1733584516914/Put/seqid=0 2024-12-07T15:15:19,250 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412076e791fcda7234cb39a50d953708bf8ed_c0d1e6963b45fdb2c1b5de83c5d92b34 store=[table=TestAcidGuarantees family=A region=c0d1e6963b45fdb2c1b5de83c5d92b34] 2024-12-07T15:15:19,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742219_1395 (size=12561) 2024-12-07T15:15:19,252 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412076e791fcda7234cb39a50d953708bf8ed_c0d1e6963b45fdb2c1b5de83c5d92b34, store=[table=TestAcidGuarantees family=A region=c0d1e6963b45fdb2c1b5de83c5d92b34] 2024-12-07T15:15:19,253 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412076e791fcda7234cb39a50d953708bf8ed_c0d1e6963b45fdb2c1b5de83c5d92b34 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c0d1e6963b45fdb2c1b5de83c5d92b34] 2024-12-07T15:15:19,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742220_1396 (size=4469) 2024-12-07T15:15:19,329 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-07T15:15:19,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:19,329 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing c0d1e6963b45fdb2c1b5de83c5d92b34 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-07T15:15:19,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0d1e6963b45fdb2c1b5de83c5d92b34, store=A 2024-12-07T15:15:19,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:15:19,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0d1e6963b45fdb2c1b5de83c5d92b34, store=B 2024-12-07T15:15:19,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:15:19,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0d1e6963b45fdb2c1b5de83c5d92b34, store=C 2024-12-07T15:15:19,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:15:19,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207b6209c8fba73470499cd4f03040f9938_c0d1e6963b45fdb2c1b5de83c5d92b34 is 50, key is test_row_0/A:col10/1733584517607/Put/seqid=0 2024-12-07T15:15:19,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742221_1397 (size=12304) 2024-12-07T15:15:19,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-07T15:15:19,658 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/B/8e0affbbc9cd436eae73d5a2d3b6ef98 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/8e0affbbc9cd436eae73d5a2d3b6ef98 2024-12-07T15:15:19,662 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0d1e6963b45fdb2c1b5de83c5d92b34#A#compaction#335 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:15:19,663 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/A/c510b3aace3242f0ae744751df483484 is 175, key is test_row_0/A:col10/1733584516914/Put/seqid=0 2024-12-07T15:15:19,664 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c0d1e6963b45fdb2c1b5de83c5d92b34/B of c0d1e6963b45fdb2c1b5de83c5d92b34 into 8e0affbbc9cd436eae73d5a2d3b6ef98(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:15:19,664 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0d1e6963b45fdb2c1b5de83c5d92b34: 2024-12-07T15:15:19,664 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., storeName=c0d1e6963b45fdb2c1b5de83c5d92b34/B, priority=13, startTime=1733584519228; duration=0sec 2024-12-07T15:15:19,664 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T15:15:19,664 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0d1e6963b45fdb2c1b5de83c5d92b34:B 2024-12-07T15:15:19,664 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:15:19,665 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:15:19,665 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1541): c0d1e6963b45fdb2c1b5de83c5d92b34/C is initiating minor compaction (all files) 2024-12-07T15:15:19,665 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0d1e6963b45fdb2c1b5de83c5d92b34/C in TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:19,665 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/bd3e663d4f6c4ae1a228f1692245ca6f, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/1993c4695bf445c28ba2a5ee972c3989, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/7f8cde578a1b471cbc15a1ea4e2a51c8] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp, totalSize=35.9 K 2024-12-07T15:15:19,666 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting bd3e663d4f6c4ae1a228f1692245ca6f, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1733584513998 2024-12-07T15:15:19,667 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 1993c4695bf445c28ba2a5ee972c3989, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1733584514703 2024-12-07T15:15:19,668 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f8cde578a1b471cbc15a1ea4e2a51c8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733584516914 2024-12-07T15:15:19,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742222_1398 (size=31515) 2024-12-07T15:15:19,674 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0d1e6963b45fdb2c1b5de83c5d92b34#C#compaction#337 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:15:19,675 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/C/512e8fb3d0a4449b9085c9de375d970e is 50, key is test_row_0/C:col10/1733584516914/Put/seqid=0 2024-12-07T15:15:19,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742223_1399 (size=12561) 2024-12-07T15:15:19,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:19,745 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207b6209c8fba73470499cd4f03040f9938_c0d1e6963b45fdb2c1b5de83c5d92b34 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207b6209c8fba73470499cd4f03040f9938_c0d1e6963b45fdb2c1b5de83c5d92b34 2024-12-07T15:15:19,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/A/b188d55a5d2c4eee87a3378c274d43eb, store: [table=TestAcidGuarantees family=A region=c0d1e6963b45fdb2c1b5de83c5d92b34] 2024-12-07T15:15:19,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/A/b188d55a5d2c4eee87a3378c274d43eb is 175, key is test_row_0/A:col10/1733584517607/Put/seqid=0 2024-12-07T15:15:19,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742224_1400 (size=31105) 2024-12-07T15:15:19,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on c0d1e6963b45fdb2c1b5de83c5d92b34 2024-12-07T15:15:19,787 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. as already flushing 2024-12-07T15:15:19,839 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:19,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42098 deadline: 1733584579825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:19,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:19,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:19,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:19,840 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:19,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42120 deadline: 1733584579826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:19,840 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:19,840 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:19,840 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:19,840 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:19,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42096 deadline: 1733584579827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:19,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:19,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:19,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42104 deadline: 1733584579828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:19,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:19,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:19,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:19,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:19,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:19,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:19,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42098 deadline: 1733584579943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:19,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:19,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:19,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:19,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:19,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42120 deadline: 1733584579943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:19,956 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:19,956 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:19,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:19,956 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:19,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42096 deadline: 1733584579943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:19,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:19,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:19,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:19,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:19,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42104 deadline: 1733584579943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:19,957 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:19,957 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:19,957 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:20,074 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/A/c510b3aace3242f0ae744751df483484 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/c510b3aace3242f0ae744751df483484 2024-12-07T15:15:20,079 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c0d1e6963b45fdb2c1b5de83c5d92b34/A of c0d1e6963b45fdb2c1b5de83c5d92b34 into c510b3aace3242f0ae744751df483484(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:15:20,079 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0d1e6963b45fdb2c1b5de83c5d92b34: 2024-12-07T15:15:20,079 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., storeName=c0d1e6963b45fdb2c1b5de83c5d92b34/A, priority=13, startTime=1733584519227; duration=0sec 2024-12-07T15:15:20,079 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:15:20,079 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0d1e6963b45fdb2c1b5de83c5d92b34:A 2024-12-07T15:15:20,082 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/C/512e8fb3d0a4449b9085c9de375d970e as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/512e8fb3d0a4449b9085c9de375d970e 2024-12-07T15:15:20,086 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c0d1e6963b45fdb2c1b5de83c5d92b34/C of c0d1e6963b45fdb2c1b5de83c5d92b34 into 512e8fb3d0a4449b9085c9de375d970e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:15:20,086 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0d1e6963b45fdb2c1b5de83c5d92b34: 2024-12-07T15:15:20,086 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., storeName=c0d1e6963b45fdb2c1b5de83c5d92b34/C, priority=13, startTime=1733584519228; duration=0sec 2024-12-07T15:15:20,086 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:15:20,086 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0d1e6963b45fdb2c1b5de83c5d92b34:C 2024-12-07T15:15:20,150 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=186, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/A/b188d55a5d2c4eee87a3378c274d43eb 2024-12-07T15:15:20,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/B/ab0d4d97a4bc4f53a94688fa7d1abfba is 50, key is test_row_0/B:col10/1733584517607/Put/seqid=0 2024-12-07T15:15:20,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742225_1401 (size=12151) 2024-12-07T15:15:20,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:20,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42098 deadline: 1733584580163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:20,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:20,174 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:20,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42120 deadline: 1733584580163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:20,174 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:20,174 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:20,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:20,175 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:20,175 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:20,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42096 deadline: 1733584580163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:20,175 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:20,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:20,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:20,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:20,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42104 deadline: 1733584580163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:20,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:20,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:20,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:20,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:20,493 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:20,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42098 deadline: 1733584580484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:20,493 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:20,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42120 deadline: 1733584580484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:20,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:20,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:20,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:20,494 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:20,494 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:20,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42096 deadline: 1733584580484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:20,494 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:20,494 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:20,494 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:20,494 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:20,494 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:20,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42104 deadline: 1733584580484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:20,494 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:20,494 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:20,494 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:20,494 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:20,561 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/B/ab0d4d97a4bc4f53a94688fa7d1abfba 2024-12-07T15:15:20,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/C/4cc6fadb35d14e5cadbb99a7bb718a56 is 50, key is test_row_0/C:col10/1733584517607/Put/seqid=0 2024-12-07T15:15:20,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742226_1402 (size=12151) 2024-12-07T15:15:20,571 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/C/4cc6fadb35d14e5cadbb99a7bb718a56 2024-12-07T15:15:20,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/A/b188d55a5d2c4eee87a3378c274d43eb as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/b188d55a5d2c4eee87a3378c274d43eb 2024-12-07T15:15:20,578 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/b188d55a5d2c4eee87a3378c274d43eb, entries=150, sequenceid=186, filesize=30.4 K 2024-12-07T15:15:20,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/B/ab0d4d97a4bc4f53a94688fa7d1abfba as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/ab0d4d97a4bc4f53a94688fa7d1abfba 2024-12-07T15:15:20,582 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/ab0d4d97a4bc4f53a94688fa7d1abfba, entries=150, sequenceid=186, filesize=11.9 K 2024-12-07T15:15:20,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/C/4cc6fadb35d14e5cadbb99a7bb718a56 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/4cc6fadb35d14e5cadbb99a7bb718a56 2024-12-07T15:15:20,586 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/4cc6fadb35d14e5cadbb99a7bb718a56, entries=150, sequenceid=186, filesize=11.9 K 2024-12-07T15:15:20,587 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for c0d1e6963b45fdb2c1b5de83c5d92b34 in 1258ms, sequenceid=186, compaction requested=false 2024-12-07T15:15:20,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for c0d1e6963b45fdb2c1b5de83c5d92b34: 2024-12-07T15:15:20,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:20,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-12-07T15:15:20,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-12-07T15:15:20,590 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-12-07T15:15:20,590 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0930 sec 2024-12-07T15:15:20,592 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 3.0970 sec 2024-12-07T15:15:21,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on c0d1e6963b45fdb2c1b5de83c5d92b34 2024-12-07T15:15:21,009 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c0d1e6963b45fdb2c1b5de83c5d92b34 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-07T15:15:21,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0d1e6963b45fdb2c1b5de83c5d92b34, store=A 2024-12-07T15:15:21,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:15:21,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0d1e6963b45fdb2c1b5de83c5d92b34, store=B 2024-12-07T15:15:21,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:15:21,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0d1e6963b45fdb2c1b5de83c5d92b34, store=C 2024-12-07T15:15:21,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:15:21,018 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207b9943c3e252b4537b1e678ba987aa1d2_c0d1e6963b45fdb2c1b5de83c5d92b34 is 50, key is test_row_0/A:col10/1733584519826/Put/seqid=0 2024-12-07T15:15:21,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742227_1403 (size=17284) 2024-12-07T15:15:21,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:21,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42120 deadline: 1733584581036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:21,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:21,049 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:21,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42104 deadline: 1733584581037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:21,049 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:21,049 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:21,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:21,049 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:21,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42096 deadline: 1733584581038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:21,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:21,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:21,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:21,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:21,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:21,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:21,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42098 deadline: 1733584581039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:21,052 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:21,052 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:21,052 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:21,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:21,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42120 deadline: 1733584581153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:21,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:21,162 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:21,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42104 deadline: 1733584581153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:21,162 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:21,162 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:21,162 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:21,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:21,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42096 deadline: 1733584581153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:21,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:21,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:21,162 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:21,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:21,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42098 deadline: 1733584581153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:21,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:21,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:21,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:21,163 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:21,163 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:21,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:21,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42120 deadline: 1733584581373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:21,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:21,388 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:21,388 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:21,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42104 deadline: 1733584581373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:21,388 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:21,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:21,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:21,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:21,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:21,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42096 deadline: 1733584581373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:21,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:21,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:21,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:21,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42098 deadline: 1733584581373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:21,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:21,390 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:21,390 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:21,390 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:21,422 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:21,429 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207b9943c3e252b4537b1e678ba987aa1d2_c0d1e6963b45fdb2c1b5de83c5d92b34 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207b9943c3e252b4537b1e678ba987aa1d2_c0d1e6963b45fdb2c1b5de83c5d92b34 2024-12-07T15:15:21,430 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/A/fe796014def14dd881320219fe5ecf51, store: [table=TestAcidGuarantees family=A region=c0d1e6963b45fdb2c1b5de83c5d92b34] 2024-12-07T15:15:21,430 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/A/fe796014def14dd881320219fe5ecf51 is 175, key is test_row_0/A:col10/1733584519826/Put/seqid=0 2024-12-07T15:15:21,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742228_1404 (size=48389) 2024-12-07T15:15:21,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-07T15:15:21,664 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T15:15:21,664 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T15:15:21,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-12-07T15:15:21,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-07T15:15:21,666 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T15:15:21,667 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T15:15:21,667 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T15:15:21,704 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:21,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42120 deadline: 1733584581693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:21,705 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:21,705 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:21,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:21,705 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:21,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42104 deadline: 1733584581693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:21,705 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:21,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:21,705 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:21,705 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:21,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42096 deadline: 1733584581693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:21,706 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:21,706 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:21,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:21,706 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:21,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42098 deadline: 1733584581693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:21,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:21,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:21,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:21,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-07T15:15:21,819 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-12-07T15:15:21,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:21,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. as already flushing 2024-12-07T15:15:21,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:21,820 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:21,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:21,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:21,838 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=207, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/A/fe796014def14dd881320219fe5ecf51 2024-12-07T15:15:21,845 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/B/5ea833ac85754f5aa5b94a63cb18d6b6 is 50, key is test_row_0/B:col10/1733584519826/Put/seqid=0 2024-12-07T15:15:21,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742229_1405 (size=12151) 2024-12-07T15:15:21,850 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/B/5ea833ac85754f5aa5b94a63cb18d6b6 2024-12-07T15:15:21,856 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/C/beddccdc4c39424a9215bbdb7788dff6 is 50, key is test_row_0/C:col10/1733584519826/Put/seqid=0 2024-12-07T15:15:21,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742230_1406 (size=12151) 2024-12-07T15:15:21,972 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-12-07T15:15:21,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:21,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. as already flushing 2024-12-07T15:15:21,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:21,973 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:21,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:21,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:21,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-07T15:15:22,125 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-12-07T15:15:22,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:22,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. as already flushing 2024-12-07T15:15:22,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:22,126 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:22,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:22,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:22,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:22,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42120 deadline: 1733584582213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:22,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:22,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42104 deadline: 1733584582214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:22,226 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:22,226 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:22,226 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:22,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:22,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42096 deadline: 1733584582214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:22,226 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:22,226 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:22,226 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:22,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:22,226 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:22,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42098 deadline: 1733584582214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:22,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:22,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:22,227 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:22,227 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:22,227 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:22,260 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/C/beddccdc4c39424a9215bbdb7788dff6 2024-12-07T15:15:22,264 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/A/fe796014def14dd881320219fe5ecf51 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/fe796014def14dd881320219fe5ecf51 2024-12-07T15:15:22,268 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/fe796014def14dd881320219fe5ecf51, entries=250, sequenceid=207, filesize=47.3 K 2024-12-07T15:15:22,268 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/B/5ea833ac85754f5aa5b94a63cb18d6b6 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/5ea833ac85754f5aa5b94a63cb18d6b6 2024-12-07T15:15:22,271 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/5ea833ac85754f5aa5b94a63cb18d6b6, entries=150, sequenceid=207, filesize=11.9 K 2024-12-07T15:15:22,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/C/beddccdc4c39424a9215bbdb7788dff6 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/beddccdc4c39424a9215bbdb7788dff6 2024-12-07T15:15:22,275 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/beddccdc4c39424a9215bbdb7788dff6, entries=150, sequenceid=207, filesize=11.9 K 2024-12-07T15:15:22,276 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for c0d1e6963b45fdb2c1b5de83c5d92b34 in 1267ms, sequenceid=207, compaction requested=true 2024-12-07T15:15:22,276 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c0d1e6963b45fdb2c1b5de83c5d92b34: 2024-12-07T15:15:22,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0d1e6963b45fdb2c1b5de83c5d92b34:A, priority=-2147483648, current under compaction store size is 1 2024-12-07T15:15:22,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:15:22,276 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:15:22,276 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:15:22,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0d1e6963b45fdb2c1b5de83c5d92b34:B, priority=-2147483648, current under compaction store size is 2 2024-12-07T15:15:22,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:15:22,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0d1e6963b45fdb2c1b5de83c5d92b34:C, priority=-2147483648, current under compaction store size is 3 2024-12-07T15:15:22,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T15:15:22,277 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:15:22,277 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111009 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:15:22,277 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1541): c0d1e6963b45fdb2c1b5de83c5d92b34/B is initiating minor compaction (all files) 2024-12-07T15:15:22,277 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1541): c0d1e6963b45fdb2c1b5de83c5d92b34/A is initiating minor compaction (all files) 2024-12-07T15:15:22,277 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0d1e6963b45fdb2c1b5de83c5d92b34/B in TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:22,277 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0d1e6963b45fdb2c1b5de83c5d92b34/A in TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:22,277 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/8e0affbbc9cd436eae73d5a2d3b6ef98, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/ab0d4d97a4bc4f53a94688fa7d1abfba, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/5ea833ac85754f5aa5b94a63cb18d6b6] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp, totalSize=36.0 K 2024-12-07T15:15:22,277 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/c510b3aace3242f0ae744751df483484, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/b188d55a5d2c4eee87a3378c274d43eb, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/fe796014def14dd881320219fe5ecf51] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp, totalSize=108.4 K 2024-12-07T15:15:22,278 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:22,278 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. files: [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/c510b3aace3242f0ae744751df483484, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/b188d55a5d2c4eee87a3378c274d43eb, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/fe796014def14dd881320219fe5ecf51] 2024-12-07T15:15:22,278 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e0affbbc9cd436eae73d5a2d3b6ef98, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733584516914 2024-12-07T15:15:22,278 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting c510b3aace3242f0ae744751df483484, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733584516914 2024-12-07T15:15:22,278 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting ab0d4d97a4bc4f53a94688fa7d1abfba, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1733584517577 2024-12-07T15:15:22,278 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-12-07T15:15:22,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:22,279 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing c0d1e6963b45fdb2c1b5de83c5d92b34 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-07T15:15:22,279 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ea833ac85754f5aa5b94a63cb18d6b6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1733584519813 2024-12-07T15:15:22,279 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting b188d55a5d2c4eee87a3378c274d43eb, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1733584517577 2024-12-07T15:15:22,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0d1e6963b45fdb2c1b5de83c5d92b34, store=A 2024-12-07T15:15:22,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:15:22,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0d1e6963b45fdb2c1b5de83c5d92b34, store=B 2024-12-07T15:15:22,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:15:22,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0d1e6963b45fdb2c1b5de83c5d92b34, store=C 2024-12-07T15:15:22,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:15:22,279 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe796014def14dd881320219fe5ecf51, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1733584519813 2024-12-07T15:15:22,291 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0d1e6963b45fdb2c1b5de83c5d92b34#B#compaction#344 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:15:22,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207a1eacb6fe9374295b69d6df38521ef57_c0d1e6963b45fdb2c1b5de83c5d92b34 is 50, key is test_row_0/A:col10/1733584521036/Put/seqid=0 2024-12-07T15:15:22,291 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/B/8fa06af5eb5642d29623e4bed117891f is 50, key is test_row_0/B:col10/1733584519826/Put/seqid=0 2024-12-07T15:15:22,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-07T15:15:22,297 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c0d1e6963b45fdb2c1b5de83c5d92b34] 2024-12-07T15:15:22,299 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120717bc09289658457ba0863db54e38b8d1_c0d1e6963b45fdb2c1b5de83c5d92b34 store=[table=TestAcidGuarantees family=A region=c0d1e6963b45fdb2c1b5de83c5d92b34] 2024-12-07T15:15:22,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742232_1408 (size=12663) 2024-12-07T15:15:22,301 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120717bc09289658457ba0863db54e38b8d1_c0d1e6963b45fdb2c1b5de83c5d92b34, store=[table=TestAcidGuarantees family=A region=c0d1e6963b45fdb2c1b5de83c5d92b34] 2024-12-07T15:15:22,302 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120717bc09289658457ba0863db54e38b8d1_c0d1e6963b45fdb2c1b5de83c5d92b34 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c0d1e6963b45fdb2c1b5de83c5d92b34] 2024-12-07T15:15:22,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742231_1407 (size=12304) 2024-12-07T15:15:22,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,308 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/B/8fa06af5eb5642d29623e4bed117891f as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/8fa06af5eb5642d29623e4bed117891f 2024-12-07T15:15:22,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742233_1409 (size=4469) 2024-12-07T15:15:22,310 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0d1e6963b45fdb2c1b5de83c5d92b34#A#compaction#345 average throughput is 1.88 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:15:22,311 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207a1eacb6fe9374295b69d6df38521ef57_c0d1e6963b45fdb2c1b5de83c5d92b34 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207a1eacb6fe9374295b69d6df38521ef57_c0d1e6963b45fdb2c1b5de83c5d92b34 2024-12-07T15:15:22,311 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/A/c24df4d4432c4d4987a3377cc1844886 is 175, key is test_row_0/A:col10/1733584519826/Put/seqid=0 2024-12-07T15:15:22,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/A/7b11ccc69eaa432a8919828b3132b4e4, store: [table=TestAcidGuarantees family=A region=c0d1e6963b45fdb2c1b5de83c5d92b34] 2024-12-07T15:15:22,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/A/7b11ccc69eaa432a8919828b3132b4e4 is 175, key is test_row_0/A:col10/1733584521036/Put/seqid=0 2024-12-07T15:15:22,315 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c0d1e6963b45fdb2c1b5de83c5d92b34/B of c0d1e6963b45fdb2c1b5de83c5d92b34 into 8fa06af5eb5642d29623e4bed117891f(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:15:22,315 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0d1e6963b45fdb2c1b5de83c5d92b34: 2024-12-07T15:15:22,315 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., storeName=c0d1e6963b45fdb2c1b5de83c5d92b34/B, priority=13, startTime=1733584522276; duration=0sec 2024-12-07T15:15:22,315 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T15:15:22,315 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0d1e6963b45fdb2c1b5de83c5d92b34:B 2024-12-07T15:15:22,315 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:15:22,316 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:15:22,316 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1541): c0d1e6963b45fdb2c1b5de83c5d92b34/C is initiating minor compaction (all files) 2024-12-07T15:15:22,316 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0d1e6963b45fdb2c1b5de83c5d92b34/C in TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:22,316 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/512e8fb3d0a4449b9085c9de375d970e, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/4cc6fadb35d14e5cadbb99a7bb718a56, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/beddccdc4c39424a9215bbdb7788dff6] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp, totalSize=36.0 K 2024-12-07T15:15:22,317 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 512e8fb3d0a4449b9085c9de375d970e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733584516914 2024-12-07T15:15:22,317 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 4cc6fadb35d14e5cadbb99a7bb718a56, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1733584517577 2024-12-07T15:15:22,317 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting beddccdc4c39424a9215bbdb7788dff6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1733584519813 2024-12-07T15:15:22,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742234_1410 (size=31105) 2024-12-07T15:15:22,323 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=221, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/A/7b11ccc69eaa432a8919828b3132b4e4 2024-12-07T15:15:22,325 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0d1e6963b45fdb2c1b5de83c5d92b34#C#compaction#346 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:15:22,326 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/C/8dc7ac5e509344efb6175110b2beb975 is 50, key is test_row_0/C:col10/1733584519826/Put/seqid=0 2024-12-07T15:15:22,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742235_1411 (size=31617) 2024-12-07T15:15:22,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742236_1412 (size=12663) 2024-12-07T15:15:22,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/B/939acf413ed74a3aac5585882f1f03c0 is 50, key is test_row_0/B:col10/1733584521036/Put/seqid=0 2024-12-07T15:15:22,336 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/A/c24df4d4432c4d4987a3377cc1844886 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/c24df4d4432c4d4987a3377cc1844886 2024-12-07T15:15:22,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742237_1413 (size=12151) 2024-12-07T15:15:22,341 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/B/939acf413ed74a3aac5585882f1f03c0 2024-12-07T15:15:22,341 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c0d1e6963b45fdb2c1b5de83c5d92b34/A of c0d1e6963b45fdb2c1b5de83c5d92b34 into c24df4d4432c4d4987a3377cc1844886(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:15:22,341 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0d1e6963b45fdb2c1b5de83c5d92b34: 2024-12-07T15:15:22,341 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., storeName=c0d1e6963b45fdb2c1b5de83c5d92b34/A, priority=13, startTime=1733584522276; duration=0sec 2024-12-07T15:15:22,341 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:15:22,341 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0d1e6963b45fdb2c1b5de83c5d92b34:A 2024-12-07T15:15:22,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/C/79c8b2a5012644158a72e1199bd67008 is 50, key is test_row_0/C:col10/1733584521036/Put/seqid=0 2024-12-07T15:15:22,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742238_1414 (size=12151) 2024-12-07T15:15:22,734 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/C/8dc7ac5e509344efb6175110b2beb975 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/8dc7ac5e509344efb6175110b2beb975 2024-12-07T15:15:22,738 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c0d1e6963b45fdb2c1b5de83c5d92b34/C of c0d1e6963b45fdb2c1b5de83c5d92b34 into 8dc7ac5e509344efb6175110b2beb975(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:15:22,739 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0d1e6963b45fdb2c1b5de83c5d92b34: 2024-12-07T15:15:22,739 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., storeName=c0d1e6963b45fdb2c1b5de83c5d92b34/C, priority=13, startTime=1733584522277; duration=0sec 2024-12-07T15:15:22,739 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:15:22,739 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0d1e6963b45fdb2c1b5de83c5d92b34:C 2024-12-07T15:15:22,755 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/C/79c8b2a5012644158a72e1199bd67008 2024-12-07T15:15:22,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/A/7b11ccc69eaa432a8919828b3132b4e4 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/7b11ccc69eaa432a8919828b3132b4e4 2024-12-07T15:15:22,764 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/7b11ccc69eaa432a8919828b3132b4e4, entries=150, sequenceid=221, filesize=30.4 K 2024-12-07T15:15:22,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/B/939acf413ed74a3aac5585882f1f03c0 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/939acf413ed74a3aac5585882f1f03c0 2024-12-07T15:15:22,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,768 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/939acf413ed74a3aac5585882f1f03c0, entries=150, sequenceid=221, filesize=11.9 K 2024-12-07T15:15:22,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/C/79c8b2a5012644158a72e1199bd67008 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/79c8b2a5012644158a72e1199bd67008 2024-12-07T15:15:22,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,772 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/79c8b2a5012644158a72e1199bd67008, entries=150, sequenceid=221, filesize=11.9 K 2024-12-07T15:15:22,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,773 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=0 B/0 for c0d1e6963b45fdb2c1b5de83c5d92b34 in 494ms, sequenceid=221, compaction requested=false 2024-12-07T15:15:22,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for c0d1e6963b45fdb2c1b5de83c5d92b34: 2024-12-07T15:15:22,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:22,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-12-07T15:15:22,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-12-07T15:15:22,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,777 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-12-07T15:15:22,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,777 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1080 sec 2024-12-07T15:15:22,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,779 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 1.1130 sec 2024-12-07T15:15:22,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-07T15:15:22,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,803 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T15:15:22,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,804 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T15:15:22,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-12-07T15:15:22,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-07T15:15:22,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,807 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T15:15:22,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,808 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T15:15:22,808 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T15:15:22,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-07T15:15:22,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,960 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-07T15:15:22,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:22,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for c0d1e6963b45fdb2c1b5de83c5d92b34: 2024-12-07T15:15:22,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:22,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-12-07T15:15:22,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-12-07T15:15:22,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,964 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-12-07T15:15:22,964 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 154 msec 2024-12-07T15:15:22,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,966 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 161 msec 2024-12-07T15:15:22,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:22,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-07T15:15:23,124 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T15:15:23,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,125 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T15:15:23,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-12-07T15:15:23,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-07T15:15:23,129 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T15:15:23,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,132 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T15:15:23,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,132 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T15:15:23,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-07T15:15:23,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,290 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-07T15:15:23,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:23,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,290 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing c0d1e6963b45fdb2c1b5de83c5d92b34 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-12-07T15:15:23,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0d1e6963b45fdb2c1b5de83c5d92b34, store=A 2024-12-07T15:15:23,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:15:23,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0d1e6963b45fdb2c1b5de83c5d92b34, store=B 2024-12-07T15:15:23,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:15:23,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0d1e6963b45fdb2c1b5de83c5d92b34, store=C 2024-12-07T15:15:23,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:15:23,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207a7315a9178b94b4b81fef79803951e0a_c0d1e6963b45fdb2c1b5de83c5d92b34 is 50, key is test_row_0/A:col10/1733584523286/Put/seqid=0 2024-12-07T15:15:23,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742239_1415 (size=12304) 2024-12-07T15:15:23,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,321 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207a7315a9178b94b4b81fef79803951e0a_c0d1e6963b45fdb2c1b5de83c5d92b34 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207a7315a9178b94b4b81fef79803951e0a_c0d1e6963b45fdb2c1b5de83c5d92b34 2024-12-07T15:15:23,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/A/118a6da0cab442438cd834f8c260fcdf, store: [table=TestAcidGuarantees family=A region=c0d1e6963b45fdb2c1b5de83c5d92b34] 2024-12-07T15:15:23,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/A/118a6da0cab442438cd834f8c260fcdf is 175, key is test_row_0/A:col10/1733584523286/Put/seqid=0 2024-12-07T15:15:23,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742240_1416 (size=31101) 2024-12-07T15:15:23,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on c0d1e6963b45fdb2c1b5de83c5d92b34 2024-12-07T15:15:23,355 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. as already flushing 2024-12-07T15:15:23,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:23,418 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733584523383, max timestamp=1733584523414Num uniques 859; 2024-12-07T15:15:23,418 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733584523383, max timestamp=1733584523414Num uniques 859; 2024-12-07T15:15:23,418 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733584523383, max timestamp=1733584523414Num uniques 859; 2024-12-07T15:15:23,418 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-07T15:15:23,418 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-07T15:15:23,418 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-07T15:15:23,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-07T15:15:23,491 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733584523447, max timestamp=1733584523485Num uniques 1068; 2024-12-07T15:15:23,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:23,491 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-12-07T15:15:23,491 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733584523447, max timestamp=1733584523485Num uniques 1068; 2024-12-07T15:15:23,491 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-12-07T15:15:23,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42104 deadline: 1733584583474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:23,491 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733584523447, max timestamp=1733584523485Num uniques 1068; 2024-12-07T15:15:23,491 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-12-07T15:15:23,492 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:23,492 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:23,492 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:23,577 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733584523529, max timestamp=1733584523572Num uniques 1027; 2024-12-07T15:15:23,577 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-12-07T15:15:23,578 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733584523529, max timestamp=1733584523572Num uniques 1027; 2024-12-07T15:15:23,578 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-12-07T15:15:23,579 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733584523529, max timestamp=1733584523572Num uniques 1027; 2024-12-07T15:15:23,579 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-12-07T15:15:23,637 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1733584523598, max timestamp=1733584523633Num uniques 1036; 2024-12-07T15:15:23,637 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=13096, suffix heap size=135712,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=286 2024-12-07T15:15:23,637 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1733584523598, max timestamp=1733584523633Num uniques 1036; 2024-12-07T15:15:23,637 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=13096, suffix heap size=135712,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=286 2024-12-07T15:15:23,640 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1733584523598, max timestamp=1733584523633Num uniques 1036; 2024-12-07T15:15:23,640 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=13096, suffix heap size=135712,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=286 2024-12-07T15:15:23,701 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1733584523659, max timestamp=1733584523694Num uniques 1145; 2024-12-07T15:15:23,701 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=11220, suffix heap size=147048,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=245 2024-12-07T15:15:23,701 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1733584523659, max timestamp=1733584523694Num uniques 1145; 2024-12-07T15:15:23,702 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1733584523659, max timestamp=1733584523694Num uniques 1145; 2024-12-07T15:15:23,702 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=11220, suffix heap size=147048,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=245 2024-12-07T15:15:23,702 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=11220, suffix heap size=147048,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=245 2024-12-07T15:15:23,732 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=231, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/A/118a6da0cab442438cd834f8c260fcdf 2024-12-07T15:15:23,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/B/26658c74b888406a8d7a4e2ac1b7638c is 50, key is test_row_0/B:col10/1733584523286/Put/seqid=0 2024-12-07T15:15:23,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742241_1417 (size=9757) 2024-12-07T15:15:23,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-07T15:15:23,778 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:23,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42104 deadline: 1733584583759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:23,778 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:23,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42098 deadline: 1733584583765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:23,780 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1733584523747, max timestamp=1733584523775Num uniques 1104; 2024-12-07T15:15:23,780 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1733584523747, max timestamp=1733584523775Num uniques 1104; 2024-12-07T15:15:23,780 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=7050, suffix heap size=142784,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=154 2024-12-07T15:15:23,780 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=7050, suffix heap size=142784,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=154 2024-12-07T15:15:23,780 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1733584523747, max timestamp=1733584523775Num uniques 1104; 2024-12-07T15:15:23,780 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=7050, suffix heap size=142784,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=154 2024-12-07T15:15:23,780 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:23,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:23,780 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:23,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:23,780 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:23,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:23,872 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1733584523837, max timestamp=1733584523868Num uniques 1013; 2024-12-07T15:15:23,872 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=7458, suffix heap size=133320,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=163 2024-12-07T15:15:23,875 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1733584523837, max timestamp=1733584523868Num uniques 1013; 2024-12-07T15:15:23,875 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=7458, suffix heap size=133320,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=163 2024-12-07T15:15:23,876 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1733584523837, max timestamp=1733584523868Num uniques 1013; 2024-12-07T15:15:23,876 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=7458, suffix heap size=133320,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=163 2024-12-07T15:15:23,935 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1733584523912, max timestamp=1733584523930Num uniques 1022; 2024-12-07T15:15:23,934 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1733584523912, max timestamp=1733584523930Num uniques 1022; 2024-12-07T15:15:23,935 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=10162, suffix heap size=134256,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=222 2024-12-07T15:15:23,935 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=10162, suffix heap size=134256,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=222 2024-12-07T15:15:23,935 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1733584523912, max timestamp=1733584523930Num uniques 1022; 2024-12-07T15:15:23,935 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=10162, suffix heap size=134256,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=222 2024-12-07T15:15:23,991 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=331, cellSize=15156, totalHeapSize=34664, min timestamp=1733584523962, max timestamp=1733584523987Num uniques 1081; 2024-12-07T15:15:23,991 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49506, new segment data size=15156, suffix heap size=140392,new segment heap size=34664 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=331 2024-12-07T15:15:23,991 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=331, cellSize=15156, totalHeapSize=34664, min timestamp=1733584523962, max timestamp=1733584523987Num uniques 1081; 2024-12-07T15:15:23,992 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49506, new segment data size=15156, suffix heap size=140392,new segment heap size=34664 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=331 2024-12-07T15:15:23,992 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=331, cellSize=15156, totalHeapSize=34664, min timestamp=1733584523962, max timestamp=1733584523987Num uniques 1081; 2024-12-07T15:15:23,992 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49506, new segment data size=15156, suffix heap size=140392,new segment heap size=34664 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=331 2024-12-07T15:15:24,047 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=290, cellSize=13280, totalHeapSize=30400, min timestamp=1733584524018, max timestamp=1733584524043Num uniques 1190; 2024-12-07T15:15:24,047 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=54500, new segment data size=13280, suffix heap size=151728,new segment heap size=30400 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1190, new segment cells count=290 2024-12-07T15:15:24,048 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=290, cellSize=13280, totalHeapSize=30400, min timestamp=1733584524018, max timestamp=1733584524043Num uniques 1190; 2024-12-07T15:15:24,048 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=54500, new segment data size=13280, suffix heap size=151728,new segment heap size=30400 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1190, new segment cells count=290 2024-12-07T15:15:24,048 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=290, cellSize=13280, totalHeapSize=30400, min timestamp=1733584524018, max timestamp=1733584524043Num uniques 1190; 2024-12-07T15:15:24,048 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=54500, new segment data size=13280, suffix heap size=151728,new segment heap size=30400 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1190, new segment cells count=290 2024-12-07T15:15:24,100 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=299, cellSize=13694, totalHeapSize=31336, min timestamp=1733584524074, max timestamp=1733584524095Num uniques 1149; 2024-12-07T15:15:24,100 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=52624, new segment data size=13694, suffix heap size=147464,new segment heap size=31336 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1149, new segment cells count=299 2024-12-07T15:15:24,100 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=349, cellSize=15984, totalHeapSize=36536, min timestamp=1733584524074, max timestamp=1733584524095Num uniques 1149; 2024-12-07T15:15:24,101 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=52624, new segment data size=15984, suffix heap size=147464,new segment heap size=36536 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1149, new segment cells count=349 2024-12-07T15:15:24,101 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=349, cellSize=15984, totalHeapSize=36536, min timestamp=1733584524074, max timestamp=1733584524095Num uniques 1149; 2024-12-07T15:15:24,101 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=52624, new segment data size=15984, suffix heap size=147464,new segment heap size=36536 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1149, new segment cells count=349 2024-12-07T15:15:24,144 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/B/26658c74b888406a8d7a4e2ac1b7638c 2024-12-07T15:15:24,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/C/c414779fd27c42068461771a42ded171 is 50, key is test_row_0/C:col10/1733584523286/Put/seqid=0 2024-12-07T15:15:24,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742242_1418 (size=9757) 2024-12-07T15:15:24,157 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/C/c414779fd27c42068461771a42ded171 2024-12-07T15:15:24,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:24,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42096 deadline: 1733584584145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:24,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:24,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:24,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:24,162 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=258, cellSize=11810, totalHeapSize=27072, min timestamp=1733584524137, max timestamp=1733584524158Num uniques 1208; 2024-12-07T15:15:24,162 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=55320, new segment data size=11810, suffix heap size=153600,new segment heap size=27072 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1208, new segment cells count=258 2024-12-07T15:15:24,163 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=258, cellSize=11810, totalHeapSize=27072, min timestamp=1733584524137, max timestamp=1733584524158Num uniques 1158; 2024-12-07T15:15:24,163 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=53030, new segment data size=11810, suffix heap size=148400,new segment heap size=27072 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1158, new segment cells count=258 2024-12-07T15:15:24,163 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=258, cellSize=11810, totalHeapSize=27072, min timestamp=1733584524137, max timestamp=1733584524158Num uniques 1208; 2024-12-07T15:15:24,163 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=55320, new segment data size=11810, suffix heap size=153600,new segment heap size=27072 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1208, new segment cells count=258 2024-12-07T15:15:24,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/A/118a6da0cab442438cd834f8c260fcdf as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/118a6da0cab442438cd834f8c260fcdf 2024-12-07T15:15:24,168 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/118a6da0cab442438cd834f8c260fcdf, entries=150, sequenceid=231, filesize=30.4 K 2024-12-07T15:15:24,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/B/26658c74b888406a8d7a4e2ac1b7638c as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/26658c74b888406a8d7a4e2ac1b7638c 2024-12-07T15:15:24,173 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/26658c74b888406a8d7a4e2ac1b7638c, entries=100, sequenceid=231, filesize=9.5 K 2024-12-07T15:15:24,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/C/c414779fd27c42068461771a42ded171 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/c414779fd27c42068461771a42ded171 2024-12-07T15:15:24,177 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/c414779fd27c42068461771a42ded171, entries=100, sequenceid=231, filesize=9.5 K 2024-12-07T15:15:24,178 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=60.38 KB/61830 for c0d1e6963b45fdb2c1b5de83c5d92b34 in 888ms, sequenceid=231, compaction requested=true 2024-12-07T15:15:24,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for c0d1e6963b45fdb2c1b5de83c5d92b34: 2024-12-07T15:15:24,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:24,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-12-07T15:15:24,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-12-07T15:15:24,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on c0d1e6963b45fdb2c1b5de83c5d92b34 2024-12-07T15:15:24,180 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c0d1e6963b45fdb2c1b5de83c5d92b34 3/3 column families, dataSize=67.09 KB heapSize=176.48 KB 2024-12-07T15:15:24,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0d1e6963b45fdb2c1b5de83c5d92b34, store=A 2024-12-07T15:15:24,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T15:15:24,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0d1e6963b45fdb2c1b5de83c5d92b34, store=B 2024-12-07T15:15:24,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T15:15:24,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0d1e6963b45fdb2c1b5de83c5d92b34, store=C 2024-12-07T15:15:24,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T15:15:24,181 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-12-07T15:15:24,181 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0470 sec 2024-12-07T15:15:24,183 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 1.0570 sec 2024-12-07T15:15:24,188 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120720ebed3dcf324f3380d17af9d66d53f2_c0d1e6963b45fdb2c1b5de83c5d92b34 is 50, key is test_row_0/A:col10/1733584524175/Put/seqid=0 2024-12-07T15:15:24,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742243_1419 (size=17534) 2024-12-07T15:15:24,260 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:24,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42098 deadline: 1733584584253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:24,261 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:24,261 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:24,261 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:24,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-07T15:15:24,264 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T15:15:24,264 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T15:15:24,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-12-07T15:15:24,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-07T15:15:24,266 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T15:15:24,267 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T15:15:24,267 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T15:15:24,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:24,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42104 deadline: 1733584584255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:24,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:24,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42120 deadline: 1733584584255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:24,275 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:24,275 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:24,275 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:24,277 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:24,277 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:24,277 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:24,283 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:24,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42096 deadline: 1733584584273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:24,285 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:24,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:24,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:24,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-07T15:15:24,376 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:24,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42098 deadline: 1733584584363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:24,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:24,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:24,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:24,382 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:24,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42136 deadline: 1733584584373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:24,383 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to d6797ebe9c52,46009,1733584418401 for 'test_row_1' in c0d1e6963b45fdb2c1b5de83c5d92b34 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18319 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-07T15:15:24,383 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:24,384 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:24,384 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:24,393 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:24,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42120 deadline: 1733584584383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:24,393 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:24,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42104 deadline: 1733584584384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:24,393 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:24,393 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:24,393 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:24,393 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:24,393 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:24,393 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:24,420 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-07T15:15:24,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:24,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. as already flushing 2024-12-07T15:15:24,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:24,420 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:24,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:24,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:24,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:24,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42096 deadline: 1733584584493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:24,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:24,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:24,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:24,573 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-07T15:15:24,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:24,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. as already flushing 2024-12-07T15:15:24,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:24,574 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:24,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:24,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:24,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-07T15:15:24,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:24,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42098 deadline: 1733584584584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:24,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:24,593 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:24,593 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:24,593 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:24,596 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120720ebed3dcf324f3380d17af9d66d53f2_c0d1e6963b45fdb2c1b5de83c5d92b34 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120720ebed3dcf324f3380d17af9d66d53f2_c0d1e6963b45fdb2c1b5de83c5d92b34 2024-12-07T15:15:24,597 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/A/5d131c8e00e54a8abda93331a64e22c4, store: [table=TestAcidGuarantees family=A region=c0d1e6963b45fdb2c1b5de83c5d92b34] 2024-12-07T15:15:24,598 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/A/5d131c8e00e54a8abda93331a64e22c4 is 175, key is test_row_0/A:col10/1733584524175/Put/seqid=0 2024-12-07T15:15:24,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742244_1420 (size=48639) 2024-12-07T15:15:24,609 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:24,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42120 deadline: 1733584584603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:24,609 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:24,609 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:24,609 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:24,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42104 deadline: 1733584584605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:24,609 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:24,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:24,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:24,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:24,727 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-07T15:15:24,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:24,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. as already flushing 2024-12-07T15:15:24,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:24,727 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:24,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:24,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:24,817 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:24,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42096 deadline: 1733584584813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:24,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:24,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:24,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:24,880 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-07T15:15:24,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:24,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. as already flushing 2024-12-07T15:15:24,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:24,880 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:24,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:24,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:24,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-07T15:15:24,912 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:24,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42098 deadline: 1733584584903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:24,913 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:24,913 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:24,913 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:24,930 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:24,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42120 deadline: 1733584584923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:24,930 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:24,930 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:24,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42104 deadline: 1733584584923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:24,930 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:24,930 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:24,930 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:24,930 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:24,930 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:25,002 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=445, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/A/5d131c8e00e54a8abda93331a64e22c4 2024-12-07T15:15:25,009 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/B/357e82b3b647414386d5f3735dd2afb4 is 50, key is test_row_0/B:col10/1733584524175/Put/seqid=0 2024-12-07T15:15:25,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742245_1421 (size=12301) 2024-12-07T15:15:25,033 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-07T15:15:25,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:25,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. as already flushing 2024-12-07T15:15:25,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:25,033 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:25,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:25,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:25,186 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-07T15:15:25,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:25,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. as already flushing 2024-12-07T15:15:25,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:25,186 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:25,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:25,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:25,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:25,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42096 deadline: 1733584585324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:25,334 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:25,334 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:25,334 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:25,339 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-07T15:15:25,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:25,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. as already flushing 2024-12-07T15:15:25,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:25,339 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:25,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:25,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:25,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-07T15:15:25,413 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=445 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/B/357e82b3b647414386d5f3735dd2afb4 2024-12-07T15:15:25,419 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/C/0c93e938dc44412db448060a260545f6 is 50, key is test_row_0/C:col10/1733584524175/Put/seqid=0 2024-12-07T15:15:25,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742246_1422 (size=12301) 2024-12-07T15:15:25,423 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=445 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/C/0c93e938dc44412db448060a260545f6 2024-12-07T15:15:25,427 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/A/5d131c8e00e54a8abda93331a64e22c4 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/5d131c8e00e54a8abda93331a64e22c4 2024-12-07T15:15:25,430 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/5d131c8e00e54a8abda93331a64e22c4, entries=250, sequenceid=445, filesize=47.5 K 2024-12-07T15:15:25,430 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/B/357e82b3b647414386d5f3735dd2afb4 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/357e82b3b647414386d5f3735dd2afb4 2024-12-07T15:15:25,432 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:25,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42098 deadline: 1733584585424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:25,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:25,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:25,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:25,435 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/357e82b3b647414386d5f3735dd2afb4, entries=150, sequenceid=445, filesize=12.0 K 2024-12-07T15:15:25,436 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/C/0c93e938dc44412db448060a260545f6 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/0c93e938dc44412db448060a260545f6 2024-12-07T15:15:25,439 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/0c93e938dc44412db448060a260545f6, entries=150, sequenceid=445, filesize=12.0 K 2024-12-07T15:15:25,440 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.44 KB/180672, currentSize=114.05 KB/116790 for c0d1e6963b45fdb2c1b5de83c5d92b34 in 1260ms, sequenceid=445, compaction requested=true 2024-12-07T15:15:25,440 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c0d1e6963b45fdb2c1b5de83c5d92b34: 2024-12-07T15:15:25,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0d1e6963b45fdb2c1b5de83c5d92b34:A, priority=-2147483648, current under compaction store size is 1 2024-12-07T15:15:25,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:15:25,440 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-07T15:15:25,441 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-07T15:15:25,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0d1e6963b45fdb2c1b5de83c5d92b34:B, priority=-2147483648, current under compaction store size is 2 2024-12-07T15:15:25,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:15:25,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0d1e6963b45fdb2c1b5de83c5d92b34:C, priority=-2147483648, current under compaction store size is 3 2024-12-07T15:15:25,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T15:15:25,441 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c0d1e6963b45fdb2c1b5de83c5d92b34 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-07T15:15:25,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0d1e6963b45fdb2c1b5de83c5d92b34, store=A 2024-12-07T15:15:25,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T15:15:25,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0d1e6963b45fdb2c1b5de83c5d92b34, store=B 2024-12-07T15:15:25,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T15:15:25,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0d1e6963b45fdb2c1b5de83c5d92b34, store=C 2024-12-07T15:15:25,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T15:15:25,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on c0d1e6963b45fdb2c1b5de83c5d92b34 2024-12-07T15:15:25,445 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46872 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-07T15:15:25,445 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1541): c0d1e6963b45fdb2c1b5de83c5d92b34/B is initiating minor compaction (all files) 2024-12-07T15:15:25,445 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0d1e6963b45fdb2c1b5de83c5d92b34/B in TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:25,445 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/8fa06af5eb5642d29623e4bed117891f, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/939acf413ed74a3aac5585882f1f03c0, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/26658c74b888406a8d7a4e2ac1b7638c, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/357e82b3b647414386d5f3735dd2afb4] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp, totalSize=45.8 K 2024-12-07T15:15:25,445 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142462 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-07T15:15:25,445 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1541): c0d1e6963b45fdb2c1b5de83c5d92b34/A is initiating minor compaction (all files) 2024-12-07T15:15:25,445 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0d1e6963b45fdb2c1b5de83c5d92b34/A in TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:25,445 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 8fa06af5eb5642d29623e4bed117891f, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1733584519813 2024-12-07T15:15:25,445 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/c24df4d4432c4d4987a3377cc1844886, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/7b11ccc69eaa432a8919828b3132b4e4, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/118a6da0cab442438cd834f8c260fcdf, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/5d131c8e00e54a8abda93331a64e22c4] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp, totalSize=139.1 K 2024-12-07T15:15:25,445 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:25,445 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. files: [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/c24df4d4432c4d4987a3377cc1844886, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/7b11ccc69eaa432a8919828b3132b4e4, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/118a6da0cab442438cd834f8c260fcdf, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/5d131c8e00e54a8abda93331a64e22c4] 2024-12-07T15:15:25,446 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 939acf413ed74a3aac5585882f1f03c0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1733584521024 2024-12-07T15:15:25,446 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting c24df4d4432c4d4987a3377cc1844886, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1733584519813 2024-12-07T15:15:25,446 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 26658c74b888406a8d7a4e2ac1b7638c, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1733584523286 2024-12-07T15:15:25,446 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b11ccc69eaa432a8919828b3132b4e4, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1733584521024 2024-12-07T15:15:25,446 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 357e82b3b647414386d5f3735dd2afb4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1733584524171 2024-12-07T15:15:25,447 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 118a6da0cab442438cd834f8c260fcdf, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1733584523286 2024-12-07T15:15:25,447 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d131c8e00e54a8abda93331a64e22c4, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1733584524137 2024-12-07T15:15:25,461 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0d1e6963b45fdb2c1b5de83c5d92b34#B#compaction#355 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:15:25,462 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/B/952269fa9c19470db80b7ced028a29d5 is 50, key is test_row_0/B:col10/1733584524175/Put/seqid=0 2024-12-07T15:15:25,463 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c0d1e6963b45fdb2c1b5de83c5d92b34] 2024-12-07T15:15:25,465 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412078a913dc2ce6e4132a89be4dfd7000da8_c0d1e6963b45fdb2c1b5de83c5d92b34 store=[table=TestAcidGuarantees family=A region=c0d1e6963b45fdb2c1b5de83c5d92b34] 2024-12-07T15:15:25,465 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120743000e5f6dbb41ac904f22f19df39221_c0d1e6963b45fdb2c1b5de83c5d92b34 is 50, key is test_row_0/A:col10/1733584524253/Put/seqid=0 2024-12-07T15:15:25,467 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412078a913dc2ce6e4132a89be4dfd7000da8_c0d1e6963b45fdb2c1b5de83c5d92b34, store=[table=TestAcidGuarantees family=A region=c0d1e6963b45fdb2c1b5de83c5d92b34] 2024-12-07T15:15:25,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742247_1423 (size=12949) 2024-12-07T15:15:25,467 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412078a913dc2ce6e4132a89be4dfd7000da8_c0d1e6963b45fdb2c1b5de83c5d92b34 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c0d1e6963b45fdb2c1b5de83c5d92b34] 2024-12-07T15:15:25,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742248_1424 (size=14994) 2024-12-07T15:15:25,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742249_1425 (size=4469) 2024-12-07T15:15:25,491 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-07T15:15:25,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:25,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. as already flushing 2024-12-07T15:15:25,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:25,492 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:25,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:25,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:25,538 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:25,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42104 deadline: 1733584585525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:25,539 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:25,539 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:25,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42120 deadline: 1733584585526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:25,539 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:25,539 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:25,539 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:25,539 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:25,539 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:25,644 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-07T15:15:25,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:25,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. as already flushing 2024-12-07T15:15:25,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:25,644 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:25,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:25,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:25,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:25,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42104 deadline: 1733584585643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:25,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:25,654 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:25,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42120 deadline: 1733584585643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:25,654 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:25,654 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:25,654 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:25,654 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:25,654 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:25,797 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-07T15:15:25,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:25,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. as already flushing 2024-12-07T15:15:25,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:25,797 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:25,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:25,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:25,870 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:25,872 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/B/952269fa9c19470db80b7ced028a29d5 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/952269fa9c19470db80b7ced028a29d5 2024-12-07T15:15:25,873 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120743000e5f6dbb41ac904f22f19df39221_c0d1e6963b45fdb2c1b5de83c5d92b34 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120743000e5f6dbb41ac904f22f19df39221_c0d1e6963b45fdb2c1b5de83c5d92b34 2024-12-07T15:15:25,874 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/A/aa1e7b3125e94cd5bfa5acd0a8bbbbca, store: [table=TestAcidGuarantees family=A region=c0d1e6963b45fdb2c1b5de83c5d92b34] 2024-12-07T15:15:25,874 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/A/aa1e7b3125e94cd5bfa5acd0a8bbbbca is 175, key is test_row_0/A:col10/1733584524253/Put/seqid=0 2024-12-07T15:15:25,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:25,878 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c0d1e6963b45fdb2c1b5de83c5d92b34/B of c0d1e6963b45fdb2c1b5de83c5d92b34 into 952269fa9c19470db80b7ced028a29d5(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:15:25,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42104 deadline: 1733584585863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:25,878 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0d1e6963b45fdb2c1b5de83c5d92b34: 2024-12-07T15:15:25,878 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., storeName=c0d1e6963b45fdb2c1b5de83c5d92b34/B, priority=12, startTime=1733584525440; duration=0sec 2024-12-07T15:15:25,878 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T15:15:25,878 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0d1e6963b45fdb2c1b5de83c5d92b34:B 2024-12-07T15:15:25,878 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-07T15:15:25,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:25,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42120 deadline: 1733584585864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:25,879 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:25,879 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:25,879 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:25,880 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:25,880 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:25,880 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:25,881 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46872 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-07T15:15:25,881 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1541): c0d1e6963b45fdb2c1b5de83c5d92b34/C is initiating minor compaction (all files) 2024-12-07T15:15:25,881 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0d1e6963b45fdb2c1b5de83c5d92b34/C in TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:25,881 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0d1e6963b45fdb2c1b5de83c5d92b34#A#compaction#356 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:15:25,882 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/A/3b977697dc8c4b6288eee94d674577c8 is 175, key is test_row_0/A:col10/1733584524175/Put/seqid=0 2024-12-07T15:15:25,882 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/8dc7ac5e509344efb6175110b2beb975, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/79c8b2a5012644158a72e1199bd67008, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/c414779fd27c42068461771a42ded171, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/0c93e938dc44412db448060a260545f6] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp, totalSize=45.8 K 2024-12-07T15:15:25,882 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 8dc7ac5e509344efb6175110b2beb975, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1733584519813 2024-12-07T15:15:25,883 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 79c8b2a5012644158a72e1199bd67008, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1733584521024 2024-12-07T15:15:25,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742250_1426 (size=39949) 2024-12-07T15:15:25,884 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting c414779fd27c42068461771a42ded171, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1733584523286 2024-12-07T15:15:25,884 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=466, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/A/aa1e7b3125e94cd5bfa5acd0a8bbbbca 2024-12-07T15:15:25,884 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c93e938dc44412db448060a260545f6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1733584524171 2024-12-07T15:15:25,890 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/B/69babcde8c5e4d7594aa6739456c575a is 50, key is test_row_0/B:col10/1733584524253/Put/seqid=0 2024-12-07T15:15:25,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742251_1427 (size=31903) 2024-12-07T15:15:25,901 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/A/3b977697dc8c4b6288eee94d674577c8 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/3b977697dc8c4b6288eee94d674577c8 2024-12-07T15:15:25,906 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0d1e6963b45fdb2c1b5de83c5d92b34#C#compaction#359 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:15:25,906 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/C/9a55c8431fdf4664afa6176aef07a216 is 50, key is test_row_0/C:col10/1733584524175/Put/seqid=0 2024-12-07T15:15:25,907 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c0d1e6963b45fdb2c1b5de83c5d92b34/A of c0d1e6963b45fdb2c1b5de83c5d92b34 into 3b977697dc8c4b6288eee94d674577c8(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:15:25,907 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0d1e6963b45fdb2c1b5de83c5d92b34: 2024-12-07T15:15:25,907 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., storeName=c0d1e6963b45fdb2c1b5de83c5d92b34/A, priority=12, startTime=1733584525440; duration=0sec 2024-12-07T15:15:25,907 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:15:25,907 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0d1e6963b45fdb2c1b5de83c5d92b34:A 2024-12-07T15:15:25,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742252_1428 (size=12301) 2024-12-07T15:15:25,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742253_1429 (size=12949) 2024-12-07T15:15:25,915 INFO [Thread-1580 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1580. 2024-12-07T15:15:25,915 DEBUG [Thread-1580 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T15:15:25,915 DEBUG [Thread-1580 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:15:25,915 DEBUG [Thread-1580 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:15:25,915 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T15:15:25,917 INFO [Thread-1588 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1588. 2024-12-07T15:15:25,917 DEBUG [Thread-1588 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T15:15:25,917 DEBUG [Thread-1588 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:15:25,917 DEBUG [Thread-1588 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:15:25,917 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T15:15:25,922 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/C/9a55c8431fdf4664afa6176aef07a216 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/9a55c8431fdf4664afa6176aef07a216 2024-12-07T15:15:25,925 INFO [Thread-1582 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1582. 2024-12-07T15:15:25,925 DEBUG [Thread-1582 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T15:15:25,925 DEBUG [Thread-1582 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:15:25,925 DEBUG [Thread-1582 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:15:25,925 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T15:15:25,926 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c0d1e6963b45fdb2c1b5de83c5d92b34/C of c0d1e6963b45fdb2c1b5de83c5d92b34 into 9a55c8431fdf4664afa6176aef07a216(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:15:25,926 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0d1e6963b45fdb2c1b5de83c5d92b34: 2024-12-07T15:15:25,926 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., storeName=c0d1e6963b45fdb2c1b5de83c5d92b34/C, priority=12, startTime=1733584525441; duration=0sec 2024-12-07T15:15:25,926 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:15:25,926 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0d1e6963b45fdb2c1b5de83c5d92b34:C 2024-12-07T15:15:25,928 INFO [Thread-1584 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1584. 2024-12-07T15:15:25,928 DEBUG [Thread-1584 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T15:15:25,928 DEBUG [Thread-1584 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:15:25,928 DEBUG [Thread-1584 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:15:25,928 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T15:15:25,932 INFO [Thread-1586 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1586. 2024-12-07T15:15:25,932 DEBUG [Thread-1586 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T15:15:25,932 DEBUG [Thread-1586 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:15:25,932 DEBUG [Thread-1586 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:15:25,932 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T15:15:25,950 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-07T15:15:25,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:25,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. as already flushing 2024-12-07T15:15:25,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:25,950 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:25,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:25,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:26,103 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-07T15:15:26,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:26,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. as already flushing 2024-12-07T15:15:26,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:26,103 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:26,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:26,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:26,183 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:26,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:26,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42104 deadline: 1733584586183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:26,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42120 deadline: 1733584586183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:26,184 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:26,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:26,184 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:26,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:26,184 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:26,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:26,256 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-07T15:15:26,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:26,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. as already flushing 2024-12-07T15:15:26,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:26,256 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:26,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:26,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:26,311 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=466 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/B/69babcde8c5e4d7594aa6739456c575a 2024-12-07T15:15:26,317 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/C/71456741b5e64507884a93fcb338acfe is 50, key is test_row_0/C:col10/1733584524253/Put/seqid=0 2024-12-07T15:15:26,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742254_1430 (size=12301) 2024-12-07T15:15:26,344 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:26,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42096 deadline: 1733584586343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:26,344 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to d6797ebe9c52,46009,1733584418401 for 'test_row_0' in c0d1e6963b45fdb2c1b5de83c5d92b34 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-07T15:15:26,345 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:26,345 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:26,345 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:26,408 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-07T15:15:26,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:26,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. as already flushing 2024-12-07T15:15:26,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:26,409 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:26,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:26,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:26,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-07T15:15:26,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:26,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42098 deadline: 1733584586443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:26,444 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to d6797ebe9c52,46009,1733584418401 for 'test_row_0' in c0d1e6963b45fdb2c1b5de83c5d92b34 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-07T15:15:26,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:26,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:26,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:26,561 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-07T15:15:26,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:26,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. as already flushing 2024-12-07T15:15:26,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:26,561 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:26,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:26,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:26,693 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:26,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:26,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42104 deadline: 1733584586693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:26,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42120 deadline: 1733584586693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:26,693 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:26,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:26,693 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:26,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:26,693 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:26,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:15:26,714 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-07T15:15:26,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:26,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. as already flushing 2024-12-07T15:15:26,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:26,714 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:26,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:26,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:26,720 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=466 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/C/71456741b5e64507884a93fcb338acfe 2024-12-07T15:15:26,723 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/A/aa1e7b3125e94cd5bfa5acd0a8bbbbca as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/aa1e7b3125e94cd5bfa5acd0a8bbbbca 2024-12-07T15:15:26,726 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/aa1e7b3125e94cd5bfa5acd0a8bbbbca, entries=200, sequenceid=466, filesize=39.0 K 2024-12-07T15:15:26,726 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/B/69babcde8c5e4d7594aa6739456c575a as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/69babcde8c5e4d7594aa6739456c575a 2024-12-07T15:15:26,729 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/69babcde8c5e4d7594aa6739456c575a, entries=150, sequenceid=466, filesize=12.0 K 2024-12-07T15:15:26,729 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/C/71456741b5e64507884a93fcb338acfe as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/71456741b5e64507884a93fcb338acfe 2024-12-07T15:15:26,732 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/71456741b5e64507884a93fcb338acfe, entries=150, sequenceid=466, filesize=12.0 K 2024-12-07T15:15:26,732 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for c0d1e6963b45fdb2c1b5de83c5d92b34 in 1291ms, sequenceid=466, compaction requested=false 2024-12-07T15:15:26,733 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c0d1e6963b45fdb2c1b5de83c5d92b34: 2024-12-07T15:15:26,867 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-07T15:15:26,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:26,867 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing c0d1e6963b45fdb2c1b5de83c5d92b34 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-07T15:15:26,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0d1e6963b45fdb2c1b5de83c5d92b34, store=A 2024-12-07T15:15:26,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:15:26,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0d1e6963b45fdb2c1b5de83c5d92b34, store=B 2024-12-07T15:15:26,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:15:26,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0d1e6963b45fdb2c1b5de83c5d92b34, store=C 2024-12-07T15:15:26,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:15:26,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412076ab0ff2bf0a74a60a41a52df4b31f5db_c0d1e6963b45fdb2c1b5de83c5d92b34 is 50, key is test_row_0/A:col10/1733584525509/Put/seqid=0 2024-12-07T15:15:26,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742255_1431 (size=12454) 2024-12-07T15:15:27,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:27,280 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412076ab0ff2bf0a74a60a41a52df4b31f5db_c0d1e6963b45fdb2c1b5de83c5d92b34 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412076ab0ff2bf0a74a60a41a52df4b31f5db_c0d1e6963b45fdb2c1b5de83c5d92b34 2024-12-07T15:15:27,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/A/8f24e6011a30462fa3783841833c3740, store: [table=TestAcidGuarantees family=A region=c0d1e6963b45fdb2c1b5de83c5d92b34] 2024-12-07T15:15:27,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/A/8f24e6011a30462fa3783841833c3740 is 175, key is test_row_0/A:col10/1733584525509/Put/seqid=0 2024-12-07T15:15:27,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742256_1432 (size=31255) 2024-12-07T15:15:27,685 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=480, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/A/8f24e6011a30462fa3783841833c3740 2024-12-07T15:15:27,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/B/aabfba97b7be4f9895b16f74bc65e37b is 50, key is test_row_0/B:col10/1733584525509/Put/seqid=0 2024-12-07T15:15:27,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742257_1433 (size=12301) 2024-12-07T15:15:27,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on c0d1e6963b45fdb2c1b5de83c5d92b34 2024-12-07T15:15:27,704 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. as already flushing 2024-12-07T15:15:27,704 INFO [Thread-1569 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1569. 2024-12-07T15:15:27,704 INFO [Thread-1575 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1575. 2024-12-07T15:15:27,704 DEBUG [Thread-1569 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T15:15:27,704 DEBUG [Thread-1569 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:15:27,704 DEBUG [Thread-1575 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T15:15:27,704 DEBUG [Thread-1575 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:15:27,705 DEBUG [Thread-1569 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:15:27,705 DEBUG [Thread-1575 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:15:27,705 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T15:15:27,705 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T15:15:28,094 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=480 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/B/aabfba97b7be4f9895b16f74bc65e37b 2024-12-07T15:15:28,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/C/5fa49708f5c14aed9dccfbc47a4f2028 is 50, key is test_row_0/C:col10/1733584525509/Put/seqid=0 2024-12-07T15:15:28,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742258_1434 (size=12301) 2024-12-07T15:15:28,355 INFO [Thread-1573 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1573. 2024-12-07T15:15:28,355 DEBUG [Thread-1573 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T15:15:28,355 DEBUG [Thread-1573 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:15:28,355 DEBUG [Thread-1573 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:15:28,355 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T15:15:28,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-07T15:15:28,455 INFO [Thread-1571 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1571. 2024-12-07T15:15:28,455 DEBUG [Thread-1571 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T15:15:28,455 DEBUG [Thread-1571 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:15:28,455 DEBUG [Thread-1571 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:15:28,455 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T15:15:28,503 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=480 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/C/5fa49708f5c14aed9dccfbc47a4f2028 2024-12-07T15:15:28,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/A/8f24e6011a30462fa3783841833c3740 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/8f24e6011a30462fa3783841833c3740 2024-12-07T15:15:28,510 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/8f24e6011a30462fa3783841833c3740, entries=150, sequenceid=480, filesize=30.5 K 2024-12-07T15:15:28,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/B/aabfba97b7be4f9895b16f74bc65e37b as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/aabfba97b7be4f9895b16f74bc65e37b 2024-12-07T15:15:28,513 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/aabfba97b7be4f9895b16f74bc65e37b, entries=150, sequenceid=480, filesize=12.0 K 2024-12-07T15:15:28,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/C/5fa49708f5c14aed9dccfbc47a4f2028 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/5fa49708f5c14aed9dccfbc47a4f2028 2024-12-07T15:15:28,516 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/5fa49708f5c14aed9dccfbc47a4f2028, entries=150, sequenceid=480, filesize=12.0 K 2024-12-07T15:15:28,517 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=26.84 KB/27480 for c0d1e6963b45fdb2c1b5de83c5d92b34 in 1649ms, sequenceid=480, compaction requested=true 2024-12-07T15:15:28,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for c0d1e6963b45fdb2c1b5de83c5d92b34: 2024-12-07T15:15:28,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:28,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-12-07T15:15:28,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-12-07T15:15:28,519 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-12-07T15:15:28,519 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 4.2510 sec 2024-12-07T15:15:28,521 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 4.2550 sec 2024-12-07T15:15:32,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-07T15:15:32,473 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T15:15:34,464 INFO [Thread-1577 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1577. 2024-12-07T15:15:34,464 DEBUG [Thread-1577 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T15:15:34,464 DEBUG [Thread-1577 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:15:34,464 DEBUG [Thread-1577 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:15:34,464 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T15:15:34,465 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-07T15:15:34,465 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 91 2024-12-07T15:15:34,465 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 100 2024-12-07T15:15:34,465 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 95 2024-12-07T15:15:34,465 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 110 2024-12-07T15:15:34,465 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 18 2024-12-07T15:15:34,465 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-07T15:15:34,465 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-07T15:15:34,465 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1061 2024-12-07T15:15:34,465 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3175 rows 2024-12-07T15:15:34,465 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1067 2024-12-07T15:15:34,465 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3182 rows 2024-12-07T15:15:34,465 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1053 2024-12-07T15:15:34,465 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3159 rows 2024-12-07T15:15:34,465 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1074 2024-12-07T15:15:34,465 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3191 rows 2024-12-07T15:15:34,465 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1069 2024-12-07T15:15:34,465 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3177 rows 2024-12-07T15:15:34,465 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-07T15:15:34,465 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-07T15:15:34,465 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:15:34,465 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:15:34,465 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T15:15:34,466 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-07T15:15:34,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-07T15:15:34,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-07T15:15:34,469 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733584534468"}]},"ts":"1733584534468"} 2024-12-07T15:15:34,470 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-07T15:15:34,470 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-07T15:15:34,470 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-07T15:15:34,471 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c0d1e6963b45fdb2c1b5de83c5d92b34, UNASSIGN}] 2024-12-07T15:15:34,472 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c0d1e6963b45fdb2c1b5de83c5d92b34, UNASSIGN 2024-12-07T15:15:34,472 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=121 updating hbase:meta row=c0d1e6963b45fdb2c1b5de83c5d92b34, regionState=CLOSING, regionLocation=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:34,474 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c0d1e6963b45fdb2c1b5de83c5d92b34, UNASSIGN because future has completed 2024-12-07T15:15:34,474 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-07T15:15:34,474 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; CloseRegionProcedure c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401}] 2024-12-07T15:15:34,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-07T15:15:34,626 INFO [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] handler.UnassignRegionHandler(122): Close c0d1e6963b45fdb2c1b5de83c5d92b34 2024-12-07T15:15:34,626 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-07T15:15:34,626 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1722): Closing c0d1e6963b45fdb2c1b5de83c5d92b34, disabling compactions & flushes 2024-12-07T15:15:34,626 INFO [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:34,626 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:34,626 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. after waiting 0 ms 2024-12-07T15:15:34,626 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:34,626 INFO [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(2902): Flushing c0d1e6963b45fdb2c1b5de83c5d92b34 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-07T15:15:34,626 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0d1e6963b45fdb2c1b5de83c5d92b34, store=A 2024-12-07T15:15:34,626 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:15:34,626 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0d1e6963b45fdb2c1b5de83c5d92b34, store=B 2024-12-07T15:15:34,626 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:15:34,626 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0d1e6963b45fdb2c1b5de83c5d92b34, store=C 2024-12-07T15:15:34,626 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:15:34,631 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120730d5535f617e4214979593bec0e048ec_c0d1e6963b45fdb2c1b5de83c5d92b34 is 50, key is test_row_0/A:col10/1733584528453/Put/seqid=0 2024-12-07T15:15:34,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742259_1435 (size=9914) 2024-12-07T15:15:34,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-07T15:15:35,035 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:35,040 INFO [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120730d5535f617e4214979593bec0e048ec_c0d1e6963b45fdb2c1b5de83c5d92b34 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120730d5535f617e4214979593bec0e048ec_c0d1e6963b45fdb2c1b5de83c5d92b34 2024-12-07T15:15:35,041 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/A/d6db0cb0a9d24212a66aca9b07770d2e, store: [table=TestAcidGuarantees family=A region=c0d1e6963b45fdb2c1b5de83c5d92b34] 2024-12-07T15:15:35,041 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/A/d6db0cb0a9d24212a66aca9b07770d2e is 175, key is test_row_0/A:col10/1733584528453/Put/seqid=0 2024-12-07T15:15:35,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742260_1436 (size=22561) 2024-12-07T15:15:35,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-07T15:15:35,446 INFO [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=488, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/A/d6db0cb0a9d24212a66aca9b07770d2e 2024-12-07T15:15:35,452 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/B/5471beb3c8ff4acba32af94f865df9d3 is 50, key is test_row_0/B:col10/1733584528453/Put/seqid=0 2024-12-07T15:15:35,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742261_1437 (size=9857) 2024-12-07T15:15:35,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-07T15:15:35,856 INFO [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=488 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/B/5471beb3c8ff4acba32af94f865df9d3 2024-12-07T15:15:35,864 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/C/312db6509e124a30a67247f0e3a0bce4 is 50, key is test_row_0/C:col10/1733584528453/Put/seqid=0 2024-12-07T15:15:35,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742262_1438 (size=9857) 2024-12-07T15:15:36,268 INFO [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=488 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/C/312db6509e124a30a67247f0e3a0bce4 2024-12-07T15:15:36,272 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/A/d6db0cb0a9d24212a66aca9b07770d2e as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/d6db0cb0a9d24212a66aca9b07770d2e 2024-12-07T15:15:36,275 INFO [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/d6db0cb0a9d24212a66aca9b07770d2e, entries=100, sequenceid=488, filesize=22.0 K 2024-12-07T15:15:36,276 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/B/5471beb3c8ff4acba32af94f865df9d3 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/5471beb3c8ff4acba32af94f865df9d3 2024-12-07T15:15:36,279 INFO [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/5471beb3c8ff4acba32af94f865df9d3, entries=100, sequenceid=488, filesize=9.6 K 2024-12-07T15:15:36,279 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/.tmp/C/312db6509e124a30a67247f0e3a0bce4 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/312db6509e124a30a67247f0e3a0bce4 2024-12-07T15:15:36,283 INFO [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/312db6509e124a30a67247f0e3a0bce4, entries=100, sequenceid=488, filesize=9.6 K 2024-12-07T15:15:36,284 INFO [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for c0d1e6963b45fdb2c1b5de83c5d92b34 in 1657ms, sequenceid=488, compaction requested=true 2024-12-07T15:15:36,284 DEBUG [StoreCloser-TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/57b9a640fdb54a228a375bbc92912932, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/3f0a391b29694c18ac1ec8470b9fff2e, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/56887c1e10dd4ccf99a5e1ad17a00400, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/962881e210e94b8ea9e71c291cb182f1, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/5e952a52845d48b983ef9dcedd0e1430, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/f99fc09bda1c45d4a4ac6d0d5aa4fc18, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/f4127c663eb842ebbcf63bfcc4dccd88, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/37fff0c09a24417daffa9325aadfe2fa, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/09815ac5f7e94af5bbcb9cd2d994d3b8, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/744219665d7741018b2945e6a9e5f1cf, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/a62ecbfb2b6b4dd9b9a9f839204f3280, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/2e21a1818f2f4126a834d2f4f70f64b2, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/c510b3aace3242f0ae744751df483484, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/b188d55a5d2c4eee87a3378c274d43eb, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/fe796014def14dd881320219fe5ecf51, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/c24df4d4432c4d4987a3377cc1844886, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/7b11ccc69eaa432a8919828b3132b4e4, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/118a6da0cab442438cd834f8c260fcdf, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/5d131c8e00e54a8abda93331a64e22c4] to archive 2024-12-07T15:15:36,285 DEBUG [StoreCloser-TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-07T15:15:36,286 DEBUG [StoreCloser-TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/57b9a640fdb54a228a375bbc92912932 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/57b9a640fdb54a228a375bbc92912932 2024-12-07T15:15:36,287 DEBUG [StoreCloser-TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/3f0a391b29694c18ac1ec8470b9fff2e to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/3f0a391b29694c18ac1ec8470b9fff2e 2024-12-07T15:15:36,288 DEBUG [StoreCloser-TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/56887c1e10dd4ccf99a5e1ad17a00400 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/56887c1e10dd4ccf99a5e1ad17a00400 2024-12-07T15:15:36,289 DEBUG [StoreCloser-TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/962881e210e94b8ea9e71c291cb182f1 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/962881e210e94b8ea9e71c291cb182f1 2024-12-07T15:15:36,290 DEBUG [StoreCloser-TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/5e952a52845d48b983ef9dcedd0e1430 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/5e952a52845d48b983ef9dcedd0e1430 2024-12-07T15:15:36,291 DEBUG [StoreCloser-TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/f99fc09bda1c45d4a4ac6d0d5aa4fc18 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/f99fc09bda1c45d4a4ac6d0d5aa4fc18 2024-12-07T15:15:36,291 DEBUG [StoreCloser-TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/f4127c663eb842ebbcf63bfcc4dccd88 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/f4127c663eb842ebbcf63bfcc4dccd88 2024-12-07T15:15:36,292 DEBUG [StoreCloser-TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/37fff0c09a24417daffa9325aadfe2fa to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/37fff0c09a24417daffa9325aadfe2fa 2024-12-07T15:15:36,293 DEBUG [StoreCloser-TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/09815ac5f7e94af5bbcb9cd2d994d3b8 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/09815ac5f7e94af5bbcb9cd2d994d3b8 2024-12-07T15:15:36,294 DEBUG [StoreCloser-TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/744219665d7741018b2945e6a9e5f1cf to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/744219665d7741018b2945e6a9e5f1cf 2024-12-07T15:15:36,295 DEBUG [StoreCloser-TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/a62ecbfb2b6b4dd9b9a9f839204f3280 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/a62ecbfb2b6b4dd9b9a9f839204f3280 2024-12-07T15:15:36,296 DEBUG [StoreCloser-TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/2e21a1818f2f4126a834d2f4f70f64b2 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/2e21a1818f2f4126a834d2f4f70f64b2 2024-12-07T15:15:36,296 DEBUG [StoreCloser-TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/c510b3aace3242f0ae744751df483484 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/c510b3aace3242f0ae744751df483484 2024-12-07T15:15:36,297 DEBUG [StoreCloser-TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/b188d55a5d2c4eee87a3378c274d43eb to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/b188d55a5d2c4eee87a3378c274d43eb 2024-12-07T15:15:36,298 DEBUG [StoreCloser-TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/fe796014def14dd881320219fe5ecf51 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/fe796014def14dd881320219fe5ecf51 2024-12-07T15:15:36,299 DEBUG [StoreCloser-TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/c24df4d4432c4d4987a3377cc1844886 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/c24df4d4432c4d4987a3377cc1844886 2024-12-07T15:15:36,300 DEBUG [StoreCloser-TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/7b11ccc69eaa432a8919828b3132b4e4 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/7b11ccc69eaa432a8919828b3132b4e4 2024-12-07T15:15:36,300 DEBUG [StoreCloser-TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/118a6da0cab442438cd834f8c260fcdf to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/118a6da0cab442438cd834f8c260fcdf 2024-12-07T15:15:36,301 DEBUG [StoreCloser-TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/5d131c8e00e54a8abda93331a64e22c4 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/5d131c8e00e54a8abda93331a64e22c4 2024-12-07T15:15:36,302 DEBUG [StoreCloser-TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/c40c8f4feb394d9f9251d3be339715f6, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/0b4d44d95ed0416492b3e38b1366a5a7, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/7b4a2eb32a644dc39ec4712a70002416, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/7c1654d6354241ebae0bcadb2adcd274, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/1df34a03f1994f9fb90408fdf78c5360, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/f16cc676892645f7b5c94ed3a12a9b54, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/348bb31c6b224c68bf1ee39bfa3c68f7, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/48802220cc5c4d73a9df4bfbf8a907a8, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/3693612937ce43228c055bb19b9f9c05, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/7283bab41cc241939ecc7356499db1f5, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/66bc69407eaf4d7ebf46f71c7701ebe9, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/8e0affbbc9cd436eae73d5a2d3b6ef98, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/c888812db65e4ee6b5650ec07a4d4ef2, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/ab0d4d97a4bc4f53a94688fa7d1abfba, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/8fa06af5eb5642d29623e4bed117891f, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/5ea833ac85754f5aa5b94a63cb18d6b6, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/939acf413ed74a3aac5585882f1f03c0, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/26658c74b888406a8d7a4e2ac1b7638c, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/357e82b3b647414386d5f3735dd2afb4] to archive 2024-12-07T15:15:36,303 DEBUG [StoreCloser-TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-07T15:15:36,304 DEBUG [StoreCloser-TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/c40c8f4feb394d9f9251d3be339715f6 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/c40c8f4feb394d9f9251d3be339715f6 2024-12-07T15:15:36,305 DEBUG [StoreCloser-TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/0b4d44d95ed0416492b3e38b1366a5a7 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/0b4d44d95ed0416492b3e38b1366a5a7 2024-12-07T15:15:36,306 DEBUG [StoreCloser-TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/7b4a2eb32a644dc39ec4712a70002416 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/7b4a2eb32a644dc39ec4712a70002416 2024-12-07T15:15:36,307 DEBUG [StoreCloser-TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/7c1654d6354241ebae0bcadb2adcd274 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/7c1654d6354241ebae0bcadb2adcd274 2024-12-07T15:15:36,308 DEBUG [StoreCloser-TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/1df34a03f1994f9fb90408fdf78c5360 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/1df34a03f1994f9fb90408fdf78c5360 2024-12-07T15:15:36,308 DEBUG [StoreCloser-TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/f16cc676892645f7b5c94ed3a12a9b54 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/f16cc676892645f7b5c94ed3a12a9b54 2024-12-07T15:15:36,309 DEBUG [StoreCloser-TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/348bb31c6b224c68bf1ee39bfa3c68f7 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/348bb31c6b224c68bf1ee39bfa3c68f7 2024-12-07T15:15:36,310 DEBUG [StoreCloser-TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/48802220cc5c4d73a9df4bfbf8a907a8 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/48802220cc5c4d73a9df4bfbf8a907a8 2024-12-07T15:15:36,311 DEBUG [StoreCloser-TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/3693612937ce43228c055bb19b9f9c05 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/3693612937ce43228c055bb19b9f9c05 2024-12-07T15:15:36,312 DEBUG [StoreCloser-TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/7283bab41cc241939ecc7356499db1f5 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/7283bab41cc241939ecc7356499db1f5 2024-12-07T15:15:36,312 DEBUG [StoreCloser-TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/66bc69407eaf4d7ebf46f71c7701ebe9 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/66bc69407eaf4d7ebf46f71c7701ebe9 2024-12-07T15:15:36,313 DEBUG [StoreCloser-TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/8e0affbbc9cd436eae73d5a2d3b6ef98 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/8e0affbbc9cd436eae73d5a2d3b6ef98 2024-12-07T15:15:36,314 DEBUG [StoreCloser-TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/c888812db65e4ee6b5650ec07a4d4ef2 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/c888812db65e4ee6b5650ec07a4d4ef2 2024-12-07T15:15:36,315 DEBUG [StoreCloser-TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/ab0d4d97a4bc4f53a94688fa7d1abfba to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/ab0d4d97a4bc4f53a94688fa7d1abfba 2024-12-07T15:15:36,316 DEBUG [StoreCloser-TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/8fa06af5eb5642d29623e4bed117891f to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/8fa06af5eb5642d29623e4bed117891f 2024-12-07T15:15:36,317 DEBUG [StoreCloser-TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/5ea833ac85754f5aa5b94a63cb18d6b6 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/5ea833ac85754f5aa5b94a63cb18d6b6 2024-12-07T15:15:36,318 DEBUG [StoreCloser-TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/939acf413ed74a3aac5585882f1f03c0 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/939acf413ed74a3aac5585882f1f03c0 2024-12-07T15:15:36,318 DEBUG [StoreCloser-TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/26658c74b888406a8d7a4e2ac1b7638c to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/26658c74b888406a8d7a4e2ac1b7638c 2024-12-07T15:15:36,319 DEBUG [StoreCloser-TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/357e82b3b647414386d5f3735dd2afb4 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/357e82b3b647414386d5f3735dd2afb4 2024-12-07T15:15:36,321 DEBUG [StoreCloser-TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/17cd0f78ce864d2cbe1bca9e6b9c7ee5, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/571c54c593ef4f89a6e0ca6d1586d1af, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/02d92415e45b4646ba7549f05930f417, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/4ec29ff85c664d84b4f597a1416efb0e, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/45b3169662524ce9a372c15c3b9f2895, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/fe357de6d4a046079722a0f4f564a873, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/94d9e76399e84a6c92746c2385aeda3c, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/3ff67f62e5e74af29ae8730b07eb391f, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/bd3e663d4f6c4ae1a228f1692245ca6f, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/a5ec1a1f1c4844f6bff28968204b23b0, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/1993c4695bf445c28ba2a5ee972c3989, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/512e8fb3d0a4449b9085c9de375d970e, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/7f8cde578a1b471cbc15a1ea4e2a51c8, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/4cc6fadb35d14e5cadbb99a7bb718a56, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/8dc7ac5e509344efb6175110b2beb975, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/beddccdc4c39424a9215bbdb7788dff6, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/79c8b2a5012644158a72e1199bd67008, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/c414779fd27c42068461771a42ded171, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/0c93e938dc44412db448060a260545f6] to archive 2024-12-07T15:15:36,321 DEBUG [StoreCloser-TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-07T15:15:36,322 DEBUG [StoreCloser-TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/17cd0f78ce864d2cbe1bca9e6b9c7ee5 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/17cd0f78ce864d2cbe1bca9e6b9c7ee5 2024-12-07T15:15:36,323 DEBUG [StoreCloser-TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/571c54c593ef4f89a6e0ca6d1586d1af to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/571c54c593ef4f89a6e0ca6d1586d1af 2024-12-07T15:15:36,324 DEBUG [StoreCloser-TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/02d92415e45b4646ba7549f05930f417 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/02d92415e45b4646ba7549f05930f417 2024-12-07T15:15:36,325 DEBUG [StoreCloser-TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/4ec29ff85c664d84b4f597a1416efb0e to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/4ec29ff85c664d84b4f597a1416efb0e 2024-12-07T15:15:36,326 DEBUG [StoreCloser-TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/45b3169662524ce9a372c15c3b9f2895 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/45b3169662524ce9a372c15c3b9f2895 2024-12-07T15:15:36,327 DEBUG [StoreCloser-TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/fe357de6d4a046079722a0f4f564a873 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/fe357de6d4a046079722a0f4f564a873 2024-12-07T15:15:36,328 DEBUG [StoreCloser-TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/94d9e76399e84a6c92746c2385aeda3c to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/94d9e76399e84a6c92746c2385aeda3c 2024-12-07T15:15:36,329 DEBUG [StoreCloser-TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/3ff67f62e5e74af29ae8730b07eb391f to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/3ff67f62e5e74af29ae8730b07eb391f 2024-12-07T15:15:36,330 DEBUG [StoreCloser-TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/bd3e663d4f6c4ae1a228f1692245ca6f to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/bd3e663d4f6c4ae1a228f1692245ca6f 2024-12-07T15:15:36,331 DEBUG [StoreCloser-TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/a5ec1a1f1c4844f6bff28968204b23b0 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/a5ec1a1f1c4844f6bff28968204b23b0 2024-12-07T15:15:36,331 DEBUG [StoreCloser-TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/1993c4695bf445c28ba2a5ee972c3989 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/1993c4695bf445c28ba2a5ee972c3989 2024-12-07T15:15:36,332 DEBUG [StoreCloser-TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/512e8fb3d0a4449b9085c9de375d970e to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/512e8fb3d0a4449b9085c9de375d970e 2024-12-07T15:15:36,333 DEBUG [StoreCloser-TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/7f8cde578a1b471cbc15a1ea4e2a51c8 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/7f8cde578a1b471cbc15a1ea4e2a51c8 2024-12-07T15:15:36,334 DEBUG [StoreCloser-TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/4cc6fadb35d14e5cadbb99a7bb718a56 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/4cc6fadb35d14e5cadbb99a7bb718a56 2024-12-07T15:15:36,335 DEBUG [StoreCloser-TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/8dc7ac5e509344efb6175110b2beb975 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/8dc7ac5e509344efb6175110b2beb975 2024-12-07T15:15:36,336 DEBUG [StoreCloser-TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/beddccdc4c39424a9215bbdb7788dff6 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/beddccdc4c39424a9215bbdb7788dff6 2024-12-07T15:15:36,337 DEBUG [StoreCloser-TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/79c8b2a5012644158a72e1199bd67008 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/79c8b2a5012644158a72e1199bd67008 2024-12-07T15:15:36,341 DEBUG [StoreCloser-TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/c414779fd27c42068461771a42ded171 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/c414779fd27c42068461771a42ded171 2024-12-07T15:15:36,347 DEBUG [StoreCloser-TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/0c93e938dc44412db448060a260545f6 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/0c93e938dc44412db448060a260545f6 2024-12-07T15:15:36,357 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/recovered.edits/491.seqid, newMaxSeqId=491, maxSeqId=4 2024-12-07T15:15:36,358 INFO [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. 2024-12-07T15:15:36,358 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1676): Region close journal for c0d1e6963b45fdb2c1b5de83c5d92b34: Waiting for close lock at 1733584534626Running coprocessor pre-close hooks at 1733584534626Disabling compacts and flushes for region at 1733584534626Disabling writes for close at 1733584534626Obtaining lock to block concurrent updates at 1733584534626Preparing flush snapshotting stores in c0d1e6963b45fdb2c1b5de83c5d92b34 at 1733584534626Finished memstore snapshotting TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1733584534626Flushing stores of TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34. at 1733584534627 (+1 ms)Flushing c0d1e6963b45fdb2c1b5de83c5d92b34/A: creating writer at 1733584534627Flushing mob file c0d1e6963b45fdb2c1b5de83c5d92b34/A: appending metadata at 1733584534631 (+4 ms)Flushing mob file c0d1e6963b45fdb2c1b5de83c5d92b34/A: closing flushed file at 1733584534631Flushing c0d1e6963b45fdb2c1b5de83c5d92b34/A: appending metadata at 1733584535041 (+410 ms)Flushing c0d1e6963b45fdb2c1b5de83c5d92b34/A: closing flushed file at 1733584535041Flushing c0d1e6963b45fdb2c1b5de83c5d92b34/B: creating writer at 1733584535449 (+408 ms)Flushing c0d1e6963b45fdb2c1b5de83c5d92b34/B: appending metadata at 1733584535452 (+3 ms)Flushing c0d1e6963b45fdb2c1b5de83c5d92b34/B: closing flushed file at 1733584535452Flushing c0d1e6963b45fdb2c1b5de83c5d92b34/C: creating writer at 1733584535860 (+408 ms)Flushing c0d1e6963b45fdb2c1b5de83c5d92b34/C: appending metadata at 1733584535863 (+3 ms)Flushing c0d1e6963b45fdb2c1b5de83c5d92b34/C: closing flushed file at 1733584535863Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4994b933: reopening flushed file at 1733584536272 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@e4492a8: reopening flushed file at 1733584536275 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1ff4e41e: reopening flushed file at 1733584536279 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for c0d1e6963b45fdb2c1b5de83c5d92b34 in 1657ms, sequenceid=488, compaction requested=true at 1733584536284 (+5 ms)Writing region close event to WAL at 1733584536348 (+64 ms)Running coprocessor post-close hooks at 1733584536358 (+10 ms)Closed at 1733584536358 2024-12-07T15:15:36,360 INFO [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] handler.UnassignRegionHandler(157): Closed c0d1e6963b45fdb2c1b5de83c5d92b34 2024-12-07T15:15:36,361 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=121 updating hbase:meta row=c0d1e6963b45fdb2c1b5de83c5d92b34, regionState=CLOSED 2024-12-07T15:15:36,363 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=122, ppid=121, state=RUNNABLE, hasLock=false; CloseRegionProcedure c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 because future has completed 2024-12-07T15:15:36,366 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-12-07T15:15:36,366 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; CloseRegionProcedure c0d1e6963b45fdb2c1b5de83c5d92b34, server=d6797ebe9c52,46009,1733584418401 in 1.8900 sec 2024-12-07T15:15:36,368 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=121, resume processing ppid=120 2024-12-07T15:15:36,368 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, ppid=120, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c0d1e6963b45fdb2c1b5de83c5d92b34, UNASSIGN in 1.8950 sec 2024-12-07T15:15:36,370 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-12-07T15:15:36,371 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8990 sec 2024-12-07T15:15:36,372 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733584536372"}]},"ts":"1733584536372"} 2024-12-07T15:15:36,374 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-07T15:15:36,374 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-07T15:15:36,376 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9080 sec 2024-12-07T15:15:36,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-07T15:15:36,614 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-07T15:15:36,614 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-07T15:15:36,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-07T15:15:36,618 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=123, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-07T15:15:36,619 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=123, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-07T15:15:36,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-07T15:15:36,623 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34 2024-12-07T15:15:36,628 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A, FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B, FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C, FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/recovered.edits] 2024-12-07T15:15:36,631 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/3b977697dc8c4b6288eee94d674577c8 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/3b977697dc8c4b6288eee94d674577c8 2024-12-07T15:15:36,632 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/8f24e6011a30462fa3783841833c3740 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/8f24e6011a30462fa3783841833c3740 2024-12-07T15:15:36,633 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/aa1e7b3125e94cd5bfa5acd0a8bbbbca to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/aa1e7b3125e94cd5bfa5acd0a8bbbbca 2024-12-07T15:15:36,635 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/d6db0cb0a9d24212a66aca9b07770d2e to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/A/d6db0cb0a9d24212a66aca9b07770d2e 2024-12-07T15:15:36,638 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/5471beb3c8ff4acba32af94f865df9d3 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/5471beb3c8ff4acba32af94f865df9d3 2024-12-07T15:15:36,639 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/69babcde8c5e4d7594aa6739456c575a to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/69babcde8c5e4d7594aa6739456c575a 2024-12-07T15:15:36,641 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/952269fa9c19470db80b7ced028a29d5 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/952269fa9c19470db80b7ced028a29d5 2024-12-07T15:15:36,646 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/aabfba97b7be4f9895b16f74bc65e37b to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/B/aabfba97b7be4f9895b16f74bc65e37b 2024-12-07T15:15:36,651 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/312db6509e124a30a67247f0e3a0bce4 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/312db6509e124a30a67247f0e3a0bce4 2024-12-07T15:15:36,653 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/5fa49708f5c14aed9dccfbc47a4f2028 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/5fa49708f5c14aed9dccfbc47a4f2028 2024-12-07T15:15:36,657 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/71456741b5e64507884a93fcb338acfe to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/71456741b5e64507884a93fcb338acfe 2024-12-07T15:15:36,661 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/9a55c8431fdf4664afa6176aef07a216 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/C/9a55c8431fdf4664afa6176aef07a216 2024-12-07T15:15:36,665 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/recovered.edits/491.seqid to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34/recovered.edits/491.seqid 2024-12-07T15:15:36,666 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/c0d1e6963b45fdb2c1b5de83c5d92b34 2024-12-07T15:15:36,666 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-07T15:15:36,667 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-07T15:15:36,668 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-07T15:15:36,676 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120720ebed3dcf324f3380d17af9d66d53f2_c0d1e6963b45fdb2c1b5de83c5d92b34 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120720ebed3dcf324f3380d17af9d66d53f2_c0d1e6963b45fdb2c1b5de83c5d92b34 2024-12-07T15:15:36,677 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412072aaa8ea424654b78b402449604d2c4f0_c0d1e6963b45fdb2c1b5de83c5d92b34 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412072aaa8ea424654b78b402449604d2c4f0_c0d1e6963b45fdb2c1b5de83c5d92b34 2024-12-07T15:15:36,679 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120730d5535f617e4214979593bec0e048ec_c0d1e6963b45fdb2c1b5de83c5d92b34 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120730d5535f617e4214979593bec0e048ec_c0d1e6963b45fdb2c1b5de83c5d92b34 2024-12-07T15:15:36,681 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412073a340af3c148469bb033779349b464b6_c0d1e6963b45fdb2c1b5de83c5d92b34 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412073a340af3c148469bb033779349b464b6_c0d1e6963b45fdb2c1b5de83c5d92b34 2024-12-07T15:15:36,682 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120743000e5f6dbb41ac904f22f19df39221_c0d1e6963b45fdb2c1b5de83c5d92b34 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120743000e5f6dbb41ac904f22f19df39221_c0d1e6963b45fdb2c1b5de83c5d92b34 2024-12-07T15:15:36,684 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412076ab0ff2bf0a74a60a41a52df4b31f5db_c0d1e6963b45fdb2c1b5de83c5d92b34 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412076ab0ff2bf0a74a60a41a52df4b31f5db_c0d1e6963b45fdb2c1b5de83c5d92b34 2024-12-07T15:15:36,685 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412079cf547757d3347ed9e9423396b7b8082_c0d1e6963b45fdb2c1b5de83c5d92b34 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412079cf547757d3347ed9e9423396b7b8082_c0d1e6963b45fdb2c1b5de83c5d92b34 2024-12-07T15:15:36,686 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207a1eacb6fe9374295b69d6df38521ef57_c0d1e6963b45fdb2c1b5de83c5d92b34 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207a1eacb6fe9374295b69d6df38521ef57_c0d1e6963b45fdb2c1b5de83c5d92b34 2024-12-07T15:15:36,687 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207a7315a9178b94b4b81fef79803951e0a_c0d1e6963b45fdb2c1b5de83c5d92b34 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207a7315a9178b94b4b81fef79803951e0a_c0d1e6963b45fdb2c1b5de83c5d92b34 2024-12-07T15:15:36,688 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207b6209c8fba73470499cd4f03040f9938_c0d1e6963b45fdb2c1b5de83c5d92b34 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207b6209c8fba73470499cd4f03040f9938_c0d1e6963b45fdb2c1b5de83c5d92b34 2024-12-07T15:15:36,688 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207b9943c3e252b4537b1e678ba987aa1d2_c0d1e6963b45fdb2c1b5de83c5d92b34 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207b9943c3e252b4537b1e678ba987aa1d2_c0d1e6963b45fdb2c1b5de83c5d92b34 2024-12-07T15:15:36,689 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207cae1fa44c0424f568ea68c336de6c18a_c0d1e6963b45fdb2c1b5de83c5d92b34 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207cae1fa44c0424f568ea68c336de6c18a_c0d1e6963b45fdb2c1b5de83c5d92b34 2024-12-07T15:15:36,690 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207cb98d10c0b3141b1b3e63fb0893f033d_c0d1e6963b45fdb2c1b5de83c5d92b34 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207cb98d10c0b3141b1b3e63fb0893f033d_c0d1e6963b45fdb2c1b5de83c5d92b34 2024-12-07T15:15:36,691 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207d27f85cb553441139e48d12df94f7634_c0d1e6963b45fdb2c1b5de83c5d92b34 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207d27f85cb553441139e48d12df94f7634_c0d1e6963b45fdb2c1b5de83c5d92b34 2024-12-07T15:15:36,693 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207dee05949fae94e76851efe56f611f31e_c0d1e6963b45fdb2c1b5de83c5d92b34 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207dee05949fae94e76851efe56f611f31e_c0d1e6963b45fdb2c1b5de83c5d92b34 2024-12-07T15:15:36,694 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207eb8d1f5b6cc248ec895a685a008cc351_c0d1e6963b45fdb2c1b5de83c5d92b34 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207eb8d1f5b6cc248ec895a685a008cc351_c0d1e6963b45fdb2c1b5de83c5d92b34 2024-12-07T15:15:36,695 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207edcaf453b13d4964bada512e85b2da37_c0d1e6963b45fdb2c1b5de83c5d92b34 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207edcaf453b13d4964bada512e85b2da37_c0d1e6963b45fdb2c1b5de83c5d92b34 2024-12-07T15:15:36,695 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-07T15:15:36,698 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=123, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-07T15:15:36,701 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-07T15:15:36,703 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-07T15:15:36,704 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=123, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-07T15:15:36,704 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-07T15:15:36,704 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733584536704"}]},"ts":"9223372036854775807"} 2024-12-07T15:15:36,706 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-07T15:15:36,706 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => c0d1e6963b45fdb2c1b5de83c5d92b34, NAME => 'TestAcidGuarantees,,1733584502757.c0d1e6963b45fdb2c1b5de83c5d92b34.', STARTKEY => '', ENDKEY => ''}] 2024-12-07T15:15:36,706 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-07T15:15:36,706 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733584536706"}]},"ts":"9223372036854775807"} 2024-12-07T15:15:36,708 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-07T15:15:36,709 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=123, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-07T15:15:36,710 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 95 msec 2024-12-07T15:15:36,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-07T15:15:36,724 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-07T15:15:36,728 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-07T15:15:36,736 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=247 (was 250), OpenFileDescriptor=451 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=438 (was 512), ProcessCount=11 (was 11), AvailableMemoryMB=6860 (was 7622) 2024-12-07T15:15:36,748 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=247, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=438, ProcessCount=11, AvailableMemoryMB=6859 2024-12-07T15:15:36,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-07T15:15:36,750 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-07T15:15:36,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] procedure2.ProcedureExecutor(1139): Stored pid=124, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-07T15:15:36,753 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-07T15:15:36,753 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:36,753 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 124 2024-12-07T15:15:36,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-12-07T15:15:36,754 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-07T15:15:36,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742263_1439 (size=1061) 2024-12-07T15:15:36,767 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd 2024-12-07T15:15:36,775 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-07T15:15:36,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742264_1440 (size=53) 2024-12-07T15:15:36,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-12-07T15:15:37,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-12-07T15:15:37,178 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T15:15:37,178 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 54b89cbce0d0fe96d7c51211398eae76, disabling compactions & flushes 2024-12-07T15:15:37,178 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:37,178 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:37,178 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. after waiting 0 ms 2024-12-07T15:15:37,178 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:37,178 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:37,178 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 54b89cbce0d0fe96d7c51211398eae76: Waiting for close lock at 1733584537178Disabling compacts and flushes for region at 1733584537178Disabling writes for close at 1733584537178Writing region close event to WAL at 1733584537178Closed at 1733584537178 2024-12-07T15:15:37,179 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-07T15:15:37,179 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733584537179"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733584537179"}]},"ts":"1733584537179"} 2024-12-07T15:15:37,181 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-07T15:15:37,181 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-07T15:15:37,181 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733584537181"}]},"ts":"1733584537181"} 2024-12-07T15:15:37,182 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-07T15:15:37,183 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=54b89cbce0d0fe96d7c51211398eae76, ASSIGN}] 2024-12-07T15:15:37,183 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=54b89cbce0d0fe96d7c51211398eae76, ASSIGN 2024-12-07T15:15:37,184 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=54b89cbce0d0fe96d7c51211398eae76, ASSIGN; state=OFFLINE, location=d6797ebe9c52,46009,1733584418401; forceNewPlan=false, retain=false 2024-12-07T15:15:37,335 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=125 updating hbase:meta row=54b89cbce0d0fe96d7c51211398eae76, regionState=OPENING, regionLocation=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:37,336 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=54b89cbce0d0fe96d7c51211398eae76, ASSIGN because future has completed 2024-12-07T15:15:37,336 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; OpenRegionProcedure 54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401}] 2024-12-07T15:15:37,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-12-07T15:15:37,490 INFO [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:37,490 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(7752): Opening region: {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} 2024-12-07T15:15:37,491 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 54b89cbce0d0fe96d7c51211398eae76 2024-12-07T15:15:37,491 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T15:15:37,491 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(7794): checking encryption for 54b89cbce0d0fe96d7c51211398eae76 2024-12-07T15:15:37,491 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(7797): checking classloading for 54b89cbce0d0fe96d7c51211398eae76 2024-12-07T15:15:37,492 INFO [StoreOpener-54b89cbce0d0fe96d7c51211398eae76-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 54b89cbce0d0fe96d7c51211398eae76 2024-12-07T15:15:37,493 INFO [StoreOpener-54b89cbce0d0fe96d7c51211398eae76-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-07T15:15:37,493 INFO [StoreOpener-54b89cbce0d0fe96d7c51211398eae76-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 54b89cbce0d0fe96d7c51211398eae76 columnFamilyName A 2024-12-07T15:15:37,493 DEBUG [StoreOpener-54b89cbce0d0fe96d7c51211398eae76-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:37,494 INFO [StoreOpener-54b89cbce0d0fe96d7c51211398eae76-1 {}] regionserver.HStore(327): Store=54b89cbce0d0fe96d7c51211398eae76/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T15:15:37,494 INFO [StoreOpener-54b89cbce0d0fe96d7c51211398eae76-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 54b89cbce0d0fe96d7c51211398eae76 2024-12-07T15:15:37,495 INFO [StoreOpener-54b89cbce0d0fe96d7c51211398eae76-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-07T15:15:37,495 INFO [StoreOpener-54b89cbce0d0fe96d7c51211398eae76-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 54b89cbce0d0fe96d7c51211398eae76 columnFamilyName B 2024-12-07T15:15:37,495 DEBUG [StoreOpener-54b89cbce0d0fe96d7c51211398eae76-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:37,495 INFO [StoreOpener-54b89cbce0d0fe96d7c51211398eae76-1 {}] regionserver.HStore(327): Store=54b89cbce0d0fe96d7c51211398eae76/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T15:15:37,495 INFO [StoreOpener-54b89cbce0d0fe96d7c51211398eae76-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 54b89cbce0d0fe96d7c51211398eae76 2024-12-07T15:15:37,496 INFO [StoreOpener-54b89cbce0d0fe96d7c51211398eae76-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-07T15:15:37,496 INFO [StoreOpener-54b89cbce0d0fe96d7c51211398eae76-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 54b89cbce0d0fe96d7c51211398eae76 columnFamilyName C 2024-12-07T15:15:37,496 DEBUG [StoreOpener-54b89cbce0d0fe96d7c51211398eae76-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:15:37,496 INFO [StoreOpener-54b89cbce0d0fe96d7c51211398eae76-1 {}] regionserver.HStore(327): Store=54b89cbce0d0fe96d7c51211398eae76/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T15:15:37,496 INFO [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:37,496 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1038): replaying wal for 54b89cbce0d0fe96d7c51211398eae76 2024-12-07T15:15:37,497 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76 2024-12-07T15:15:37,497 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76 2024-12-07T15:15:37,498 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1048): stopping wal replay for 54b89cbce0d0fe96d7c51211398eae76 2024-12-07T15:15:37,498 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1060): Cleaning up temporary data for 54b89cbce0d0fe96d7c51211398eae76 2024-12-07T15:15:37,498 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-07T15:15:37,499 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1093): writing seq id for 54b89cbce0d0fe96d7c51211398eae76 2024-12-07T15:15:37,500 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T15:15:37,500 INFO [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1114): Opened 54b89cbce0d0fe96d7c51211398eae76; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70675651, jitterRate=0.0531492680311203}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-07T15:15:37,500 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 54b89cbce0d0fe96d7c51211398eae76 2024-12-07T15:15:37,501 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1006): Region open journal for 54b89cbce0d0fe96d7c51211398eae76: Running coprocessor pre-open hook at 1733584537491Writing region info on filesystem at 1733584537491Initializing all the Stores at 1733584537492 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733584537492Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733584537492Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733584537492Cleaning up temporary data from old regions at 1733584537498 (+6 ms)Running coprocessor post-open hooks at 1733584537500 (+2 ms)Region opened successfully at 1733584537501 (+1 ms) 2024-12-07T15:15:37,502 INFO [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., pid=126, masterSystemTime=1733584537488 2024-12-07T15:15:37,503 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:37,503 INFO [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:37,503 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=125 updating hbase:meta row=54b89cbce0d0fe96d7c51211398eae76, regionState=OPEN, openSeqNum=2, regionLocation=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:37,504 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=126, ppid=125, state=RUNNABLE, hasLock=false; OpenRegionProcedure 54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 because future has completed 2024-12-07T15:15:37,506 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-12-07T15:15:37,506 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; OpenRegionProcedure 54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 in 169 msec 2024-12-07T15:15:37,507 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=125, resume processing ppid=124 2024-12-07T15:15:37,507 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, ppid=124, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=54b89cbce0d0fe96d7c51211398eae76, ASSIGN in 323 msec 2024-12-07T15:15:37,508 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-07T15:15:37,508 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733584537508"}]},"ts":"1733584537508"} 2024-12-07T15:15:37,509 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-07T15:15:37,510 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-07T15:15:37,511 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 760 msec 2024-12-07T15:15:37,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-12-07T15:15:37,893 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-07T15:15:37,894 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ded0f15, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:15:37,894 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6797ebe9c52,36915,-1 for getting cluster id 2024-12-07T15:15:37,894 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T15:15:37,895 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f8956274-ed33-4e16-82eb-d5d5911ec893' 2024-12-07T15:15:37,896 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T15:15:37,896 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f8956274-ed33-4e16-82eb-d5d5911ec893" 2024-12-07T15:15:37,896 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@267cebb4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:15:37,896 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6797ebe9c52,36915,-1] 2024-12-07T15:15:37,896 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T15:15:37,896 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:15:37,897 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53670, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T15:15:37,897 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d39ae47, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:15:37,898 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T15:15:37,898 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6797ebe9c52,46009,1733584418401, seqNum=-1] 2024-12-07T15:15:37,898 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T15:15:37,899 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43888, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T15:15:37,900 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-07T15:15:37,900 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is d6797ebe9c52,36915,1733584417704 2024-12-07T15:15:37,900 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7ed78e7b 2024-12-07T15:15:37,900 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-07T15:15:37,901 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53680, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-07T15:15:37,902 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d6d1845, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:15:37,902 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6797ebe9c52,36915,-1 for getting cluster id 2024-12-07T15:15:37,902 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T15:15:37,902 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f8956274-ed33-4e16-82eb-d5d5911ec893' 2024-12-07T15:15:37,903 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T15:15:37,903 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f8956274-ed33-4e16-82eb-d5d5911ec893" 2024-12-07T15:15:37,903 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6998d2ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:15:37,903 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6797ebe9c52,36915,-1] 2024-12-07T15:15:37,903 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T15:15:37,903 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:15:37,904 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53708, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T15:15:37,904 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5aaa755d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:15:37,904 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@503e1f3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:15:37,905 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6797ebe9c52,36915,-1 for getting cluster id 2024-12-07T15:15:37,905 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T15:15:37,905 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f8956274-ed33-4e16-82eb-d5d5911ec893' 2024-12-07T15:15:37,905 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T15:15:37,905 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f8956274-ed33-4e16-82eb-d5d5911ec893" 2024-12-07T15:15:37,905 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@713e0068, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:15:37,905 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6797ebe9c52,36915,-1] 2024-12-07T15:15:37,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T15:15:37,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:15:37,906 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53724, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T15:15:37,907 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a166962, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:15:37,907 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@205b33d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:15:37,907 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6797ebe9c52,36915,-1 for getting cluster id 2024-12-07T15:15:37,907 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T15:15:37,908 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f8956274-ed33-4e16-82eb-d5d5911ec893' 2024-12-07T15:15:37,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T15:15:37,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f8956274-ed33-4e16-82eb-d5d5911ec893" 2024-12-07T15:15:37,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2291d181, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:15:37,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6797ebe9c52,36915,-1] 2024-12-07T15:15:37,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T15:15:37,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:15:37,909 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53736, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T15:15:37,909 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69c110a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:15:37,910 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e100fd5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:15:37,910 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6797ebe9c52,36915,-1 for getting cluster id 2024-12-07T15:15:37,910 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T15:15:37,910 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f8956274-ed33-4e16-82eb-d5d5911ec893' 2024-12-07T15:15:37,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T15:15:37,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f8956274-ed33-4e16-82eb-d5d5911ec893" 2024-12-07T15:15:37,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8d46fe9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:15:37,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6797ebe9c52,36915,-1] 2024-12-07T15:15:37,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T15:15:37,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:15:37,911 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53750, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T15:15:37,912 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11123a77, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:15:37,912 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58e40afb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:15:37,912 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6797ebe9c52,36915,-1 for getting cluster id 2024-12-07T15:15:37,912 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T15:15:37,913 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f8956274-ed33-4e16-82eb-d5d5911ec893' 2024-12-07T15:15:37,913 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T15:15:37,913 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f8956274-ed33-4e16-82eb-d5d5911ec893" 2024-12-07T15:15:37,913 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c1dcb91, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:15:37,913 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6797ebe9c52,36915,-1] 2024-12-07T15:15:37,913 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T15:15:37,913 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:15:37,914 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53758, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T15:15:37,914 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20d2122b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:15:37,915 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@259a56de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:15:37,915 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6797ebe9c52,36915,-1 for getting cluster id 2024-12-07T15:15:37,915 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T15:15:37,915 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f8956274-ed33-4e16-82eb-d5d5911ec893' 2024-12-07T15:15:37,915 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T15:15:37,915 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f8956274-ed33-4e16-82eb-d5d5911ec893" 2024-12-07T15:15:37,916 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1da89039, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:15:37,916 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6797ebe9c52,36915,-1] 2024-12-07T15:15:37,916 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T15:15:37,916 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:15:37,916 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53768, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T15:15:37,917 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b73e8e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:15:37,917 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e4cbf6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:15:37,917 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6797ebe9c52,36915,-1 for getting cluster id 2024-12-07T15:15:37,917 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T15:15:37,918 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f8956274-ed33-4e16-82eb-d5d5911ec893' 2024-12-07T15:15:37,918 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T15:15:37,918 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f8956274-ed33-4e16-82eb-d5d5911ec893" 2024-12-07T15:15:37,918 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11a1b17e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:15:37,918 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6797ebe9c52,36915,-1] 2024-12-07T15:15:37,918 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T15:15:37,918 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:15:37,919 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53772, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T15:15:37,919 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c40bf6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:15:37,920 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c83334c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:15:37,920 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6797ebe9c52,36915,-1 for getting cluster id 2024-12-07T15:15:37,920 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T15:15:37,921 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f8956274-ed33-4e16-82eb-d5d5911ec893' 2024-12-07T15:15:37,921 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T15:15:37,921 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f8956274-ed33-4e16-82eb-d5d5911ec893" 2024-12-07T15:15:37,921 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69ebd96c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:15:37,921 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6797ebe9c52,36915,-1] 2024-12-07T15:15:37,921 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T15:15:37,921 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:15:37,922 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53780, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T15:15:37,922 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33f925c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:15:37,922 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3532b5c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:15:37,923 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6797ebe9c52,36915,-1 for getting cluster id 2024-12-07T15:15:37,923 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T15:15:37,923 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f8956274-ed33-4e16-82eb-d5d5911ec893' 2024-12-07T15:15:37,923 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T15:15:37,923 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f8956274-ed33-4e16-82eb-d5d5911ec893" 2024-12-07T15:15:37,923 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@681ad832, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:15:37,923 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6797ebe9c52,36915,-1] 2024-12-07T15:15:37,924 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T15:15:37,924 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:15:37,924 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53798, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T15:15:37,925 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@698b7938, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:15:37,925 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3088fab0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:15:37,925 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6797ebe9c52,36915,-1 for getting cluster id 2024-12-07T15:15:37,925 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T15:15:37,926 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f8956274-ed33-4e16-82eb-d5d5911ec893' 2024-12-07T15:15:37,926 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T15:15:37,926 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f8956274-ed33-4e16-82eb-d5d5911ec893" 2024-12-07T15:15:37,926 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@179aedee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:15:37,926 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6797ebe9c52,36915,-1] 2024-12-07T15:15:37,926 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T15:15:37,926 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:15:37,927 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53806, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T15:15:37,927 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76c3dcc2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:15:37,928 DEBUG [Thread-1992 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T15:15:37,928 DEBUG [Thread-1977 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T15:15:37,928 DEBUG [Thread-1988 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T15:15:37,928 DEBUG [Thread-1979 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T15:15:37,928 DEBUG [Thread-1981 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T15:15:37,928 DEBUG [Thread-1986 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T15:15:37,929 DEBUG [Thread-1975 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T15:15:37,929 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6797ebe9c52,46009,1733584418401, seqNum=-1] 2024-12-07T15:15:37,929 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6797ebe9c52,46009,1733584418401, seqNum=-1] 2024-12-07T15:15:37,929 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6797ebe9c52,46009,1733584418401, seqNum=-1] 2024-12-07T15:15:37,929 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T15:15:37,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6797ebe9c52,46009,1733584418401, seqNum=-1] 2024-12-07T15:15:37,929 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T15:15:37,929 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6797ebe9c52,46009,1733584418401, seqNum=-1] 2024-12-07T15:15:37,929 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6797ebe9c52,46009,1733584418401, seqNum=-1] 2024-12-07T15:15:37,929 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T15:15:37,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T15:15:37,929 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T15:15:37,929 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T15:15:37,930 DEBUG [Thread-1983 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T15:15:37,930 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6797ebe9c52,46009,1733584418401, seqNum=-1] 2024-12-07T15:15:37,930 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T15:15:37,931 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43894, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T15:15:37,931 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43912, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T15:15:37,931 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43896, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T15:15:37,931 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43926, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T15:15:37,931 DEBUG [Thread-1994 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T15:15:37,931 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43936, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T15:15:37,931 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43952, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T15:15:37,932 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43928, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T15:15:37,932 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T15:15:37,932 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6797ebe9c52,46009,1733584418401, seqNum=-1] 2024-12-07T15:15:37,932 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6797ebe9c52,46009,1733584418401, seqNum=-1] 2024-12-07T15:15:37,932 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T15:15:37,932 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T15:15:37,933 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2] 2024-12-07T15:15:37,933 DEBUG [Thread-1990 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T15:15:37,933 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2] 2024-12-07T15:15:37,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-12-07T15:15:37,933 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2] 2024-12-07T15:15:37,934 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43956, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T15:15:37,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2] 2024-12-07T15:15:37,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-07T15:15:37,935 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6797ebe9c52,46009,1733584418401, seqNum=-1] 2024-12-07T15:15:37,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2] 2024-12-07T15:15:37,935 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T15:15:37,935 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43962, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T15:15:37,935 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2] 2024-12-07T15:15:37,935 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2] 2024-12-07T15:15:37,936 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43974, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T15:15:37,936 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T15:15:37,937 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2] 2024-12-07T15:15:37,937 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2] 2024-12-07T15:15:37,937 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T15:15:37,937 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T15:15:37,938 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2] 2024-12-07T15:15:37,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on 54b89cbce0d0fe96d7c51211398eae76 2024-12-07T15:15:37,944 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 54b89cbce0d0fe96d7c51211398eae76 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-07T15:15:37,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b89cbce0d0fe96d7c51211398eae76, store=A 2024-12-07T15:15:37,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:15:37,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b89cbce0d0fe96d7c51211398eae76, store=B 2024-12-07T15:15:37,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:15:37,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b89cbce0d0fe96d7c51211398eae76, store=C 2024-12-07T15:15:37,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:15:37,968 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/A/dabb52f8fc0e40d68fb52038c6b5d63d is 50, key is test_row_0/A:col10/1733584537944/Put/seqid=0 2024-12-07T15:15:37,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742265_1441 (size=12001) 2024-12-07T15:15:37,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:37,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43952 deadline: 1733584597979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:37,991 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:37,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:37,991 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:37,991 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:37,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43912 deadline: 1733584597980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:37,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:37,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:37,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:37,994 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:37,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1733584597981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:37,995 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:37,995 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:37,995 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:37,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43956 deadline: 1733584597982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:37,995 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:37,995 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:37,995 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:37,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43928 deadline: 1733584597983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:37,995 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:37,995 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:37,995 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:37,995 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:37,996 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:38,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-07T15:15:38,090 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-07T15:15:38,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:38,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. as already flushing 2024-12-07T15:15:38,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:38,091 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:38,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:38,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:38,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:38,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43952 deadline: 1733584598093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:38,101 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:38,101 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:38,101 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:38,109 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:38,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43912 deadline: 1733584598103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:38,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:38,110 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:38,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:38,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:38,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1733584598103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:38,110 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:38,110 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:38,110 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:38,110 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:38,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43956 deadline: 1733584598103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:38,110 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:38,110 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:38,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43928 deadline: 1733584598103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:38,111 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:38,111 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:38,111 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:38,111 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:38,111 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:38,243 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-07T15:15:38,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:38,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. as already flushing 2024-12-07T15:15:38,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:38,244 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:38,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:38,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:38,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-07T15:15:38,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:38,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43952 deadline: 1733584598313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:38,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:38,319 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:38,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43912 deadline: 1733584598313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:38,319 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:38,319 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:38,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:38,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:38,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:38,320 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:38,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1733584598313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:38,320 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:38,320 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:38,320 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:38,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:38,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43956 deadline: 1733584598313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:38,320 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:38,320 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:38,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:38,320 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:38,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43928 deadline: 1733584598313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:38,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:38,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:38,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:38,372 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/A/dabb52f8fc0e40d68fb52038c6b5d63d 2024-12-07T15:15:38,396 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-07T15:15:38,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:38,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. as already flushing 2024-12-07T15:15:38,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:38,397 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:38,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:38,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:38,398 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/B/56a77bcc7b8c49f2b0f0ab4e5209b3fc is 50, key is test_row_0/B:col10/1733584537944/Put/seqid=0 2024-12-07T15:15:38,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742266_1442 (size=12001) 2024-12-07T15:15:38,402 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/B/56a77bcc7b8c49f2b0f0ab4e5209b3fc 2024-12-07T15:15:38,426 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/C/ba0de81359b348508b24c616eb8dd827 is 50, key is test_row_0/C:col10/1733584537944/Put/seqid=0 2024-12-07T15:15:38,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742267_1443 (size=12001) 2024-12-07T15:15:38,549 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-07T15:15:38,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:38,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. as already flushing 2024-12-07T15:15:38,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:38,549 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:38,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:38,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:38,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-07T15:15:38,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:38,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43952 deadline: 1733584598624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:38,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:38,628 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:38,628 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:38,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43912 deadline: 1733584598624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:38,628 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:38,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:38,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:38,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:38,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:38,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1733584598624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:38,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:38,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:38,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:38,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:38,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43956 deadline: 1733584598624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:38,634 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:38,634 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:38,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:38,634 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:38,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43928 deadline: 1733584598624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:38,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:38,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:38,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:38,702 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-07T15:15:38,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:38,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. as already flushing 2024-12-07T15:15:38,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:38,702 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:38,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:38,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:38,830 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/C/ba0de81359b348508b24c616eb8dd827 2024-12-07T15:15:38,835 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/A/dabb52f8fc0e40d68fb52038c6b5d63d as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/dabb52f8fc0e40d68fb52038c6b5d63d 2024-12-07T15:15:38,838 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/dabb52f8fc0e40d68fb52038c6b5d63d, entries=150, sequenceid=13, filesize=11.7 K 2024-12-07T15:15:38,839 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/B/56a77bcc7b8c49f2b0f0ab4e5209b3fc as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/56a77bcc7b8c49f2b0f0ab4e5209b3fc 2024-12-07T15:15:38,842 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/56a77bcc7b8c49f2b0f0ab4e5209b3fc, entries=150, sequenceid=13, filesize=11.7 K 2024-12-07T15:15:38,843 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/C/ba0de81359b348508b24c616eb8dd827 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/ba0de81359b348508b24c616eb8dd827 2024-12-07T15:15:38,846 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/ba0de81359b348508b24c616eb8dd827, entries=150, sequenceid=13, filesize=11.7 K 2024-12-07T15:15:38,846 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 54b89cbce0d0fe96d7c51211398eae76 in 902ms, sequenceid=13, compaction requested=false 2024-12-07T15:15:38,846 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 54b89cbce0d0fe96d7c51211398eae76: 2024-12-07T15:15:38,855 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-07T15:15:38,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:38,855 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 54b89cbce0d0fe96d7c51211398eae76 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-07T15:15:38,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b89cbce0d0fe96d7c51211398eae76, store=A 2024-12-07T15:15:38,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:15:38,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b89cbce0d0fe96d7c51211398eae76, store=B 2024-12-07T15:15:38,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:15:38,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b89cbce0d0fe96d7c51211398eae76, store=C 2024-12-07T15:15:38,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:15:38,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/A/923f834ced3840c1b0756b4749e64a80 is 50, key is test_row_0/A:col10/1733584537982/Put/seqid=0 2024-12-07T15:15:38,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742268_1444 (size=12001) 2024-12-07T15:15:39,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-07T15:15:39,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on 54b89cbce0d0fe96d7c51211398eae76 2024-12-07T15:15:39,137 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. as already flushing 2024-12-07T15:15:39,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:39,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43952 deadline: 1733584599149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:39,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:39,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1733584599152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:39,160 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:39,160 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:39,160 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:39,161 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:39,161 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:39,161 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:39,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:39,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43956 deadline: 1733584599153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:39,166 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:39,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:39,166 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:39,166 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:39,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43928 deadline: 1733584599154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:39,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:39,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:39,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:39,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:39,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43912 deadline: 1733584599155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:39,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:39,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:39,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:39,264 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/A/923f834ced3840c1b0756b4749e64a80 2024-12-07T15:15:39,267 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:39,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43952 deadline: 1733584599263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:39,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:39,268 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:39,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1733584599263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:39,268 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:39,268 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:39,268 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:39,268 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:39,268 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:39,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/B/ceeed6b9995a49efbc7b8fa54be5c384 is 50, key is test_row_0/B:col10/1733584537982/Put/seqid=0 2024-12-07T15:15:39,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742269_1445 (size=12001) 2024-12-07T15:15:39,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:39,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43956 deadline: 1733584599273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:39,280 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:39,280 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:39,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:39,280 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:39,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43928 deadline: 1733584599273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:39,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:39,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43912 deadline: 1733584599273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:39,281 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:39,281 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:39,281 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:39,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:39,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:39,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:39,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:39,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43952 deadline: 1733584599473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:39,477 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:39,477 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:39,477 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:39,477 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:39,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1733584599473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:39,477 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:39,477 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:39,477 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:39,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:39,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43928 deadline: 1733584599483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:39,487 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:39,487 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:39,487 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:39,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:39,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43956 deadline: 1733584599493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:39,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:39,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43912 deadline: 1733584599493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:39,497 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:39,497 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:39,498 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:39,498 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:39,498 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:39,498 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:39,675 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/B/ceeed6b9995a49efbc7b8fa54be5c384 2024-12-07T15:15:39,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/C/f3998da92c3742729539b9d54ee959bb is 50, key is test_row_0/C:col10/1733584537982/Put/seqid=0 2024-12-07T15:15:39,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742270_1446 (size=12001) 2024-12-07T15:15:39,790 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:39,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43952 deadline: 1733584599783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:39,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:39,791 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:39,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1733584599783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:39,791 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:39,791 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:39,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:39,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:39,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:39,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:39,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43928 deadline: 1733584599793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:39,804 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:39,804 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:39,804 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:39,809 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:39,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43956 deadline: 1733584599803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:39,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:39,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43912 deadline: 1733584599803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:39,810 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:39,810 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:39,810 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:39,810 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:39,810 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:39,810 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:40,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-07T15:15:40,098 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/C/f3998da92c3742729539b9d54ee959bb 2024-12-07T15:15:40,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/A/923f834ced3840c1b0756b4749e64a80 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/923f834ced3840c1b0756b4749e64a80 2024-12-07T15:15:40,106 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/923f834ced3840c1b0756b4749e64a80, entries=150, sequenceid=33, filesize=11.7 K 2024-12-07T15:15:40,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/B/ceeed6b9995a49efbc7b8fa54be5c384 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/ceeed6b9995a49efbc7b8fa54be5c384 2024-12-07T15:15:40,111 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/ceeed6b9995a49efbc7b8fa54be5c384, entries=150, sequenceid=33, filesize=11.7 K 2024-12-07T15:15:40,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/C/f3998da92c3742729539b9d54ee959bb as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/f3998da92c3742729539b9d54ee959bb 2024-12-07T15:15:40,115 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/f3998da92c3742729539b9d54ee959bb, entries=150, sequenceid=33, filesize=11.7 K 2024-12-07T15:15:40,116 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 54b89cbce0d0fe96d7c51211398eae76 in 1261ms, sequenceid=33, compaction requested=false 2024-12-07T15:15:40,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 54b89cbce0d0fe96d7c51211398eae76: 2024-12-07T15:15:40,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:40,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-12-07T15:15:40,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-12-07T15:15:40,119 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-12-07T15:15:40,119 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1800 sec 2024-12-07T15:15:40,121 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 2.1880 sec 2024-12-07T15:15:40,172 WARN [regionserver/d6797ebe9c52:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 2, running: 1 2024-12-07T15:15:40,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on 54b89cbce0d0fe96d7c51211398eae76 2024-12-07T15:15:40,309 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 54b89cbce0d0fe96d7c51211398eae76 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-07T15:15:40,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b89cbce0d0fe96d7c51211398eae76, store=A 2024-12-07T15:15:40,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:15:40,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b89cbce0d0fe96d7c51211398eae76, store=B 2024-12-07T15:15:40,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:15:40,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b89cbce0d0fe96d7c51211398eae76, store=C 2024-12-07T15:15:40,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:15:40,314 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/A/5646c131a0dc44338639daf197194ef7 is 50, key is test_row_0/A:col10/1733584540307/Put/seqid=0 2024-12-07T15:15:40,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742271_1447 (size=14341) 2024-12-07T15:15:40,353 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:40,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1733584600339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:40,353 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:40,353 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:40,353 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:40,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43928 deadline: 1733584600341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:40,353 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:40,353 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:40,354 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:40,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43912 deadline: 1733584600344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:40,354 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:40,354 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:40,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:40,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:40,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:40,356 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:40,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43956 deadline: 1733584600352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:40,356 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:40,356 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:40,356 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:40,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:40,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43952 deadline: 1733584600353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:40,367 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:40,367 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:40,367 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:40,473 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:40,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1733584600463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:40,473 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:40,473 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:40,473 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:40,474 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:40,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43928 deadline: 1733584600463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:40,474 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:40,474 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:40,474 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:40,474 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:40,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43912 deadline: 1733584600463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:40,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:40,474 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:40,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:40,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:40,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43956 deadline: 1733584600463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:40,474 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:40,474 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:40,474 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:40,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:40,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43952 deadline: 1733584600473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:40,480 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:40,480 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:40,480 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:40,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:40,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1733584600683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:40,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:40,693 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:40,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43928 deadline: 1733584600683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:40,693 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:40,693 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:40,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:40,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:40,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:40,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43912 deadline: 1733584600684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:40,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:40,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:40,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:40,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:40,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43956 deadline: 1733584600684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:40,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:40,694 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:40,694 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:40,694 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:40,694 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:40,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43952 deadline: 1733584600684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:40,694 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:40,694 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:40,694 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:40,717 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/A/5646c131a0dc44338639daf197194ef7 2024-12-07T15:15:40,724 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/B/70042830246648acae6493cc1efdd302 is 50, key is test_row_0/B:col10/1733584540307/Put/seqid=0 2024-12-07T15:15:40,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742272_1448 (size=12001) 2024-12-07T15:15:40,728 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/B/70042830246648acae6493cc1efdd302 2024-12-07T15:15:40,734 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/C/f941a5cebd264600973c0c29a38c8c87 is 50, key is test_row_0/C:col10/1733584540307/Put/seqid=0 2024-12-07T15:15:40,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742273_1449 (size=12001) 2024-12-07T15:15:41,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:41,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1733584601003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:41,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:41,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43928 deadline: 1733584601003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:41,012 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:41,012 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:41,012 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:41,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:41,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:41,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43912 deadline: 1733584601004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:41,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:41,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:41,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:41,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:41,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:41,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43956 deadline: 1733584601004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:41,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:41,013 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:41,013 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:41,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:41,013 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:41,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43952 deadline: 1733584601004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:41,013 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:41,014 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:41,014 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:41,138 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/C/f941a5cebd264600973c0c29a38c8c87 2024-12-07T15:15:41,142 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/A/5646c131a0dc44338639daf197194ef7 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/5646c131a0dc44338639daf197194ef7 2024-12-07T15:15:41,145 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/5646c131a0dc44338639daf197194ef7, entries=200, sequenceid=46, filesize=14.0 K 2024-12-07T15:15:41,146 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/B/70042830246648acae6493cc1efdd302 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/70042830246648acae6493cc1efdd302 2024-12-07T15:15:41,149 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/70042830246648acae6493cc1efdd302, entries=150, sequenceid=46, filesize=11.7 K 2024-12-07T15:15:41,149 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/C/f941a5cebd264600973c0c29a38c8c87 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/f941a5cebd264600973c0c29a38c8c87 2024-12-07T15:15:41,153 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/f941a5cebd264600973c0c29a38c8c87, entries=150, sequenceid=46, filesize=11.7 K 2024-12-07T15:15:41,154 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 54b89cbce0d0fe96d7c51211398eae76 in 845ms, sequenceid=46, compaction requested=true 2024-12-07T15:15:41,154 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 54b89cbce0d0fe96d7c51211398eae76: 2024-12-07T15:15:41,155 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54b89cbce0d0fe96d7c51211398eae76:A, priority=-2147483648, current under compaction store size is 1 2024-12-07T15:15:41,155 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:15:41,155 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54b89cbce0d0fe96d7c51211398eae76:B, priority=-2147483648, current under compaction store size is 2 2024-12-07T15:15:41,155 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:15:41,155 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:15:41,155 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54b89cbce0d0fe96d7c51211398eae76:C, priority=-2147483648, current under compaction store size is 3 2024-12-07T15:15:41,155 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:15:41,155 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T15:15:41,155 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:15:41,155 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:15:41,155 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1541): 54b89cbce0d0fe96d7c51211398eae76/A is initiating minor compaction (all files) 2024-12-07T15:15:41,155 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1541): 54b89cbce0d0fe96d7c51211398eae76/B is initiating minor compaction (all files) 2024-12-07T15:15:41,156 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54b89cbce0d0fe96d7c51211398eae76/A in TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:41,156 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54b89cbce0d0fe96d7c51211398eae76/B in TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:41,156 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/dabb52f8fc0e40d68fb52038c6b5d63d, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/923f834ced3840c1b0756b4749e64a80, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/5646c131a0dc44338639daf197194ef7] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp, totalSize=37.4 K 2024-12-07T15:15:41,156 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/56a77bcc7b8c49f2b0f0ab4e5209b3fc, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/ceeed6b9995a49efbc7b8fa54be5c384, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/70042830246648acae6493cc1efdd302] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp, totalSize=35.2 K 2024-12-07T15:15:41,156 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting dabb52f8fc0e40d68fb52038c6b5d63d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733584537942 2024-12-07T15:15:41,156 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 56a77bcc7b8c49f2b0f0ab4e5209b3fc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733584537942 2024-12-07T15:15:41,156 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 923f834ced3840c1b0756b4749e64a80, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1733584537978 2024-12-07T15:15:41,156 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting ceeed6b9995a49efbc7b8fa54be5c384, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1733584537978 2024-12-07T15:15:41,156 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5646c131a0dc44338639daf197194ef7, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1733584539140 2024-12-07T15:15:41,156 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 70042830246648acae6493cc1efdd302, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1733584539140 2024-12-07T15:15:41,162 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54b89cbce0d0fe96d7c51211398eae76#A#compaction#376 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:15:41,163 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/A/d1eb582c009d48f2a0ac18eca652e901 is 50, key is test_row_0/A:col10/1733584540307/Put/seqid=0 2024-12-07T15:15:41,165 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54b89cbce0d0fe96d7c51211398eae76#B#compaction#377 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:15:41,166 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/B/6578588c544142e282d7c3001c6eebf2 is 50, key is test_row_0/B:col10/1733584540307/Put/seqid=0 2024-12-07T15:15:41,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742274_1450 (size=12104) 2024-12-07T15:15:41,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742275_1451 (size=12104) 2024-12-07T15:15:41,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on 54b89cbce0d0fe96d7c51211398eae76 2024-12-07T15:15:41,531 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 54b89cbce0d0fe96d7c51211398eae76 3/3 column families, dataSize=114.05 KB heapSize=339.89 KB 2024-12-07T15:15:41,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b89cbce0d0fe96d7c51211398eae76, store=A 2024-12-07T15:15:41,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T15:15:41,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b89cbce0d0fe96d7c51211398eae76, store=B 2024-12-07T15:15:41,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T15:15:41,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b89cbce0d0fe96d7c51211398eae76, store=C 2024-12-07T15:15:41,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T15:15:41,545 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/A/1634e8062e774003a6c355d3f2747ecf is 50, key is test_row_0/A:col10/1733584540342/Put/seqid=0 2024-12-07T15:15:41,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742276_1452 (size=16681) 2024-12-07T15:15:41,549 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=67 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/A/1634e8062e774003a6c355d3f2747ecf 2024-12-07T15:15:41,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:41,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1733584601546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:41,553 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:41,553 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:41,553 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:41,555 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/B/4749a1e92688422480c5513df39eec29 is 50, key is test_row_0/B:col10/1733584540342/Put/seqid=0 2024-12-07T15:15:41,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742277_1453 (size=12001) 2024-12-07T15:15:41,559 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=67 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/B/4749a1e92688422480c5513df39eec29 2024-12-07T15:15:41,564 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:41,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43928 deadline: 1733584601547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:41,564 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/C/57a4deeb863c4fe58ee0cbe3fea400ef is 50, key is test_row_0/C:col10/1733584540342/Put/seqid=0 2024-12-07T15:15:41,565 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:41,565 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:41,565 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:41,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:41,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43912 deadline: 1733584601549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:41,567 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:41,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43956 deadline: 1733584601549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:41,568 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:41,568 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:41,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:41,568 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:41,568 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:41,568 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:41,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43952 deadline: 1733584601550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:41,568 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:41,569 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:41,569 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:41,569 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:41,572 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/A/d1eb582c009d48f2a0ac18eca652e901 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/d1eb582c009d48f2a0ac18eca652e901 2024-12-07T15:15:41,578 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54b89cbce0d0fe96d7c51211398eae76/A of 54b89cbce0d0fe96d7c51211398eae76 into d1eb582c009d48f2a0ac18eca652e901(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:15:41,578 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54b89cbce0d0fe96d7c51211398eae76: 2024-12-07T15:15:41,578 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., storeName=54b89cbce0d0fe96d7c51211398eae76/A, priority=13, startTime=1733584541154; duration=0sec 2024-12-07T15:15:41,578 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T15:15:41,578 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54b89cbce0d0fe96d7c51211398eae76:A 2024-12-07T15:15:41,578 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:15:41,579 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:15:41,579 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1541): 54b89cbce0d0fe96d7c51211398eae76/C is initiating minor compaction (all files) 2024-12-07T15:15:41,579 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54b89cbce0d0fe96d7c51211398eae76/C in TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:41,579 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/ba0de81359b348508b24c616eb8dd827, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/f3998da92c3742729539b9d54ee959bb, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/f941a5cebd264600973c0c29a38c8c87] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp, totalSize=35.2 K 2024-12-07T15:15:41,580 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba0de81359b348508b24c616eb8dd827, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733584537942 2024-12-07T15:15:41,580 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting f3998da92c3742729539b9d54ee959bb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1733584537978 2024-12-07T15:15:41,580 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting f941a5cebd264600973c0c29a38c8c87, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1733584539140 2024-12-07T15:15:41,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742278_1454 (size=12001) 2024-12-07T15:15:41,587 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=67 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/C/57a4deeb863c4fe58ee0cbe3fea400ef 2024-12-07T15:15:41,590 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/A/1634e8062e774003a6c355d3f2747ecf as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/1634e8062e774003a6c355d3f2747ecf 2024-12-07T15:15:41,593 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/1634e8062e774003a6c355d3f2747ecf, entries=250, sequenceid=67, filesize=16.3 K 2024-12-07T15:15:41,594 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/B/4749a1e92688422480c5513df39eec29 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/4749a1e92688422480c5513df39eec29 2024-12-07T15:15:41,595 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/B/6578588c544142e282d7c3001c6eebf2 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/6578588c544142e282d7c3001c6eebf2 2024-12-07T15:15:41,597 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/4749a1e92688422480c5513df39eec29, entries=150, sequenceid=67, filesize=11.7 K 2024-12-07T15:15:41,598 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/C/57a4deeb863c4fe58ee0cbe3fea400ef as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/57a4deeb863c4fe58ee0cbe3fea400ef 2024-12-07T15:15:41,600 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54b89cbce0d0fe96d7c51211398eae76#C#compaction#381 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:15:41,600 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54b89cbce0d0fe96d7c51211398eae76/B of 54b89cbce0d0fe96d7c51211398eae76 into 6578588c544142e282d7c3001c6eebf2(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:15:41,600 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54b89cbce0d0fe96d7c51211398eae76: 2024-12-07T15:15:41,600 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., storeName=54b89cbce0d0fe96d7c51211398eae76/B, priority=13, startTime=1733584541155; duration=0sec 2024-12-07T15:15:41,600 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:15:41,600 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54b89cbce0d0fe96d7c51211398eae76:B 2024-12-07T15:15:41,601 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/C/a7474607a4ed4bbda251d1c0639795fb is 50, key is test_row_0/C:col10/1733584540307/Put/seqid=0 2024-12-07T15:15:41,605 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/57a4deeb863c4fe58ee0cbe3fea400ef, entries=150, sequenceid=67, filesize=11.7 K 2024-12-07T15:15:41,605 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 54b89cbce0d0fe96d7c51211398eae76 in 74ms, sequenceid=67, compaction requested=false 2024-12-07T15:15:41,605 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 54b89cbce0d0fe96d7c51211398eae76: 2024-12-07T15:15:41,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742279_1455 (size=12104) 2024-12-07T15:15:41,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on 54b89cbce0d0fe96d7c51211398eae76 2024-12-07T15:15:41,672 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 54b89cbce0d0fe96d7c51211398eae76 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-07T15:15:41,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b89cbce0d0fe96d7c51211398eae76, store=A 2024-12-07T15:15:41,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:15:41,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b89cbce0d0fe96d7c51211398eae76, store=B 2024-12-07T15:15:41,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:15:41,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b89cbce0d0fe96d7c51211398eae76, store=C 2024-12-07T15:15:41,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:15:41,678 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/A/176d80eb058c4138849e38eb9b6d9698 is 50, key is test_row_0/A:col10/1733584541548/Put/seqid=0 2024-12-07T15:15:41,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742280_1456 (size=14341) 2024-12-07T15:15:41,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:41,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1733584601716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:41,731 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:41,731 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:41,731 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:41,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:41,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43952 deadline: 1733584601717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:41,731 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:41,731 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:41,731 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:41,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:41,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43956 deadline: 1733584601718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:41,732 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:41,732 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:41,732 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:41,734 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:41,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43912 deadline: 1733584601719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:41,734 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:41,734 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:41,734 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:41,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:41,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43928 deadline: 1733584601731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:41,745 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:41,745 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:41,745 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:41,840 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:41,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1733584601833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:41,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:41,841 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:41,841 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:41,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43956 deadline: 1733584601834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:41,841 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:41,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:41,841 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:41,841 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:41,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43952 deadline: 1733584601834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:41,841 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:41,841 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:41,841 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:41,841 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:41,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:41,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43912 deadline: 1733584601843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:41,854 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:41,854 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:41,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:41,862 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:41,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43928 deadline: 1733584601853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:41,862 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:41,863 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:41,863 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:41,880 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-07T15:15:42,016 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/C/a7474607a4ed4bbda251d1c0639795fb as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/a7474607a4ed4bbda251d1c0639795fb 2024-12-07T15:15:42,020 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54b89cbce0d0fe96d7c51211398eae76/C of 54b89cbce0d0fe96d7c51211398eae76 into a7474607a4ed4bbda251d1c0639795fb(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:15:42,020 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54b89cbce0d0fe96d7c51211398eae76: 2024-12-07T15:15:42,020 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., storeName=54b89cbce0d0fe96d7c51211398eae76/C, priority=13, startTime=1733584541155; duration=0sec 2024-12-07T15:15:42,020 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:15:42,020 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54b89cbce0d0fe96d7c51211398eae76:C 2024-12-07T15:15:42,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:42,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1733584602053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:42,063 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:42,063 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:42,064 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:42,064 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:42,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43956 deadline: 1733584602053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:42,064 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:42,064 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:42,064 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:42,064 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:42,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43952 deadline: 1733584602053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:42,065 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:42,065 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:42,065 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:42,071 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:42,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43912 deadline: 1733584602063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:42,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:42,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:42,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:42,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:42,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43928 deadline: 1733584602073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:42,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:42,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:42,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:42,086 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/A/176d80eb058c4138849e38eb9b6d9698 2024-12-07T15:15:42,093 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/B/6dcc0c54c6db447b8a652852eb319469 is 50, key is test_row_0/B:col10/1733584541548/Put/seqid=0 2024-12-07T15:15:42,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742281_1457 (size=12001) 2024-12-07T15:15:42,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-07T15:15:42,104 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T15:15:42,104 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T15:15:42,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-12-07T15:15:42,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-07T15:15:42,106 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T15:15:42,106 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T15:15:42,107 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T15:15:42,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-07T15:15:42,259 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-07T15:15:42,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:42,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. as already flushing 2024-12-07T15:15:42,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:42,259 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:42,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:42,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:42,381 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:42,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1733584602374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:42,382 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:42,382 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:42,382 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:42,382 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:42,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43952 deadline: 1733584602374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:42,382 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:42,382 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:42,382 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:42,382 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:42,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43956 deadline: 1733584602374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:42,383 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:42,383 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:42,383 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:42,391 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:42,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43912 deadline: 1733584602383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:42,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:42,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:42,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:42,391 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:42,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43928 deadline: 1733584602383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:42,392 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:42,392 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:42,392 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:42,412 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-07T15:15:42,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:42,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. as already flushing 2024-12-07T15:15:42,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:42,412 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:42,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:42,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:42,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-07T15:15:42,497 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/B/6dcc0c54c6db447b8a652852eb319469 2024-12-07T15:15:42,510 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/C/eca82b3a850549a99957e609ebd19d85 is 50, key is test_row_0/C:col10/1733584541548/Put/seqid=0 2024-12-07T15:15:42,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742282_1458 (size=12001) 2024-12-07T15:15:42,565 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-07T15:15:42,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:42,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. as already flushing 2024-12-07T15:15:42,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:42,565 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:42,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:42,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:42,718 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-07T15:15:42,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:42,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. as already flushing 2024-12-07T15:15:42,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:42,718 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:42,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:42,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:42,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-07T15:15:42,871 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-07T15:15:42,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:42,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. as already flushing 2024-12-07T15:15:42,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:42,871 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:42,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:42,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:42,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:42,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1733584602893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:42,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:42,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:42,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43952 deadline: 1733584602893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:42,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:42,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:42,901 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:42,901 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:42,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43956 deadline: 1733584602893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:42,901 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:42,901 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:42,901 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:42,901 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:42,901 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:42,913 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:42,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43912 deadline: 1733584602904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:42,913 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:42,913 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:42,913 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:42,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43928 deadline: 1733584602904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:42,913 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:42,913 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:42,913 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:42,913 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:42,916 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/C/eca82b3a850549a99957e609ebd19d85 2024-12-07T15:15:42,920 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/A/176d80eb058c4138849e38eb9b6d9698 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/176d80eb058c4138849e38eb9b6d9698 2024-12-07T15:15:42,923 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/176d80eb058c4138849e38eb9b6d9698, entries=200, sequenceid=81, filesize=14.0 K 2024-12-07T15:15:42,924 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/B/6dcc0c54c6db447b8a652852eb319469 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/6dcc0c54c6db447b8a652852eb319469 2024-12-07T15:15:42,927 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/6dcc0c54c6db447b8a652852eb319469, entries=150, sequenceid=81, filesize=11.7 K 2024-12-07T15:15:42,928 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/C/eca82b3a850549a99957e609ebd19d85 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/eca82b3a850549a99957e609ebd19d85 2024-12-07T15:15:42,930 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/eca82b3a850549a99957e609ebd19d85, entries=150, sequenceid=81, filesize=11.7 K 2024-12-07T15:15:42,931 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 54b89cbce0d0fe96d7c51211398eae76 in 1259ms, sequenceid=81, compaction requested=true 2024-12-07T15:15:42,931 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 54b89cbce0d0fe96d7c51211398eae76: 2024-12-07T15:15:42,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54b89cbce0d0fe96d7c51211398eae76:A, priority=-2147483648, current under compaction store size is 1 2024-12-07T15:15:42,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:15:42,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54b89cbce0d0fe96d7c51211398eae76:B, priority=-2147483648, current under compaction store size is 2 2024-12-07T15:15:42,932 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:15:42,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:15:42,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54b89cbce0d0fe96d7c51211398eae76:C, priority=-2147483648, current under compaction store size is 3 2024-12-07T15:15:42,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T15:15:42,932 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:15:42,932 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43126 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:15:42,932 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:15:42,932 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1541): 54b89cbce0d0fe96d7c51211398eae76/A is initiating minor compaction (all files) 2024-12-07T15:15:42,932 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1541): 54b89cbce0d0fe96d7c51211398eae76/B is initiating minor compaction (all files) 2024-12-07T15:15:42,932 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54b89cbce0d0fe96d7c51211398eae76/A in TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:42,932 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54b89cbce0d0fe96d7c51211398eae76/B in TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:42,933 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/d1eb582c009d48f2a0ac18eca652e901, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/1634e8062e774003a6c355d3f2747ecf, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/176d80eb058c4138849e38eb9b6d9698] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp, totalSize=42.1 K 2024-12-07T15:15:42,933 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/6578588c544142e282d7c3001c6eebf2, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/4749a1e92688422480c5513df39eec29, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/6dcc0c54c6db447b8a652852eb319469] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp, totalSize=35.3 K 2024-12-07T15:15:42,933 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting d1eb582c009d48f2a0ac18eca652e901, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1733584539140 2024-12-07T15:15:42,933 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 6578588c544142e282d7c3001c6eebf2, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1733584539140 2024-12-07T15:15:42,933 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1634e8062e774003a6c355d3f2747ecf, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=67, earliestPutTs=1733584540335 2024-12-07T15:15:42,933 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 4749a1e92688422480c5513df39eec29, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=67, earliestPutTs=1733584540342 2024-12-07T15:15:42,933 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 176d80eb058c4138849e38eb9b6d9698, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1733584541547 2024-12-07T15:15:42,934 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 6dcc0c54c6db447b8a652852eb319469, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1733584541547 2024-12-07T15:15:42,941 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54b89cbce0d0fe96d7c51211398eae76#A#compaction#385 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:15:42,941 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/A/6c20ccda6d184c62b3ffe21d39b56d97 is 50, key is test_row_0/A:col10/1733584541548/Put/seqid=0 2024-12-07T15:15:42,941 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54b89cbce0d0fe96d7c51211398eae76#B#compaction#386 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:15:42,942 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/B/ff8575e113df49d89e98792faf4f01aa is 50, key is test_row_0/B:col10/1733584541548/Put/seqid=0 2024-12-07T15:15:42,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742283_1459 (size=12207) 2024-12-07T15:15:42,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742284_1460 (size=12207) 2024-12-07T15:15:42,951 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/B/ff8575e113df49d89e98792faf4f01aa as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/ff8575e113df49d89e98792faf4f01aa 2024-12-07T15:15:42,952 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/A/6c20ccda6d184c62b3ffe21d39b56d97 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/6c20ccda6d184c62b3ffe21d39b56d97 2024-12-07T15:15:42,958 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54b89cbce0d0fe96d7c51211398eae76/A of 54b89cbce0d0fe96d7c51211398eae76 into 6c20ccda6d184c62b3ffe21d39b56d97(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:15:42,958 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54b89cbce0d0fe96d7c51211398eae76: 2024-12-07T15:15:42,958 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., storeName=54b89cbce0d0fe96d7c51211398eae76/A, priority=13, startTime=1733584542931; duration=0sec 2024-12-07T15:15:42,959 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T15:15:42,959 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54b89cbce0d0fe96d7c51211398eae76:A 2024-12-07T15:15:42,959 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54b89cbce0d0fe96d7c51211398eae76/B of 54b89cbce0d0fe96d7c51211398eae76 into ff8575e113df49d89e98792faf4f01aa(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:15:42,959 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54b89cbce0d0fe96d7c51211398eae76: 2024-12-07T15:15:42,959 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:15:42,959 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., storeName=54b89cbce0d0fe96d7c51211398eae76/B, priority=13, startTime=1733584542932; duration=0sec 2024-12-07T15:15:42,959 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:15:42,959 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54b89cbce0d0fe96d7c51211398eae76:B 2024-12-07T15:15:42,960 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:15:42,960 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1541): 54b89cbce0d0fe96d7c51211398eae76/C is initiating minor compaction (all files) 2024-12-07T15:15:42,960 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54b89cbce0d0fe96d7c51211398eae76/C in TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:42,960 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/a7474607a4ed4bbda251d1c0639795fb, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/57a4deeb863c4fe58ee0cbe3fea400ef, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/eca82b3a850549a99957e609ebd19d85] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp, totalSize=35.3 K 2024-12-07T15:15:42,960 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting a7474607a4ed4bbda251d1c0639795fb, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1733584539140 2024-12-07T15:15:42,961 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 57a4deeb863c4fe58ee0cbe3fea400ef, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=67, earliestPutTs=1733584540342 2024-12-07T15:15:42,961 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting eca82b3a850549a99957e609ebd19d85, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1733584541547 2024-12-07T15:15:42,967 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54b89cbce0d0fe96d7c51211398eae76#C#compaction#387 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:15:42,968 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/C/de53b8864ad944a1bb39c0f7830cf6d0 is 50, key is test_row_0/C:col10/1733584541548/Put/seqid=0 2024-12-07T15:15:42,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742285_1461 (size=12207) 2024-12-07T15:15:43,024 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-07T15:15:43,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:43,024 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 54b89cbce0d0fe96d7c51211398eae76 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-07T15:15:43,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b89cbce0d0fe96d7c51211398eae76, store=A 2024-12-07T15:15:43,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:15:43,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b89cbce0d0fe96d7c51211398eae76, store=B 2024-12-07T15:15:43,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:15:43,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b89cbce0d0fe96d7c51211398eae76, store=C 2024-12-07T15:15:43,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:15:43,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/A/a5cf5ec83742473594991e6ed0e89f4b is 50, key is test_row_0/A:col10/1733584541716/Put/seqid=0 2024-12-07T15:15:43,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742286_1462 (size=12001) 2024-12-07T15:15:43,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-07T15:15:43,375 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/C/de53b8864ad944a1bb39c0f7830cf6d0 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/de53b8864ad944a1bb39c0f7830cf6d0 2024-12-07T15:15:43,379 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54b89cbce0d0fe96d7c51211398eae76/C of 54b89cbce0d0fe96d7c51211398eae76 into de53b8864ad944a1bb39c0f7830cf6d0(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:15:43,379 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54b89cbce0d0fe96d7c51211398eae76: 2024-12-07T15:15:43,379 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., storeName=54b89cbce0d0fe96d7c51211398eae76/C, priority=13, startTime=1733584542932; duration=0sec 2024-12-07T15:15:43,379 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:15:43,379 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54b89cbce0d0fe96d7c51211398eae76:C 2024-12-07T15:15:43,436 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/A/a5cf5ec83742473594991e6ed0e89f4b 2024-12-07T15:15:43,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/B/ccb1ba5153064582ad94672f1b720ef1 is 50, key is test_row_0/B:col10/1733584541716/Put/seqid=0 2024-12-07T15:15:43,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742287_1463 (size=12001) 2024-12-07T15:15:43,846 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/B/ccb1ba5153064582ad94672f1b720ef1 2024-12-07T15:15:43,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/C/3a06ff06cc7c4b5c81e43799748b6a7c is 50, key is test_row_0/C:col10/1733584541716/Put/seqid=0 2024-12-07T15:15:43,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742288_1464 (size=12001) 2024-12-07T15:15:43,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on 54b89cbce0d0fe96d7c51211398eae76 2024-12-07T15:15:43,919 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. as already flushing 2024-12-07T15:15:43,935 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:43,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1733584603931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:43,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:43,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:43,936 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:43,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:43,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43952 deadline: 1733584603931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:43,936 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:43,936 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:43,936 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:43,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:43,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:43,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43912 deadline: 1733584603933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:43,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43928 deadline: 1733584603934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:43,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:43,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:43,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:43,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:43,948 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:43,948 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:43,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43956 deadline: 1733584603935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:43,948 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:43,949 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:43,949 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:43,949 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:44,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:44,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1733584604043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:44,049 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:44,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:44,049 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:44,049 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:44,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43952 deadline: 1733584604043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:44,049 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:44,049 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:44,049 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:44,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:44,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43912 deadline: 1733584604053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:44,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:44,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:44,062 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:44,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:44,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43928 deadline: 1733584604053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:44,062 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:44,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:44,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:44,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43956 deadline: 1733584604053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:44,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:44,063 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:44,063 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:44,063 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:44,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-07T15:15:44,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:44,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1733584604253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:44,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:44,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:44,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:44,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:44,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43952 deadline: 1733584604253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:44,258 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:44,258 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:44,258 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:44,260 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/C/3a06ff06cc7c4b5c81e43799748b6a7c 2024-12-07T15:15:44,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/A/a5cf5ec83742473594991e6ed0e89f4b as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/a5cf5ec83742473594991e6ed0e89f4b 2024-12-07T15:15:44,267 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/a5cf5ec83742473594991e6ed0e89f4b, entries=150, sequenceid=104, filesize=11.7 K 2024-12-07T15:15:44,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/B/ccb1ba5153064582ad94672f1b720ef1 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/ccb1ba5153064582ad94672f1b720ef1 2024-12-07T15:15:44,270 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/ccb1ba5153064582ad94672f1b720ef1, entries=150, sequenceid=104, filesize=11.7 K 2024-12-07T15:15:44,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/C/3a06ff06cc7c4b5c81e43799748b6a7c as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/3a06ff06cc7c4b5c81e43799748b6a7c 2024-12-07T15:15:44,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:44,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43912 deadline: 1733584604274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:44,279 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:44,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:44,279 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:44,279 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:44,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43928 deadline: 1733584604274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:44,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:44,280 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:44,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43956 deadline: 1733584604274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:44,280 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:44,280 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:44,280 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:44,280 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:44,280 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:44,286 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/3a06ff06cc7c4b5c81e43799748b6a7c, entries=150, sequenceid=104, filesize=11.7 K 2024-12-07T15:15:44,287 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 54b89cbce0d0fe96d7c51211398eae76 in 1263ms, sequenceid=104, compaction requested=false 2024-12-07T15:15:44,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 54b89cbce0d0fe96d7c51211398eae76: 2024-12-07T15:15:44,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:44,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-12-07T15:15:44,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-12-07T15:15:44,291 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-12-07T15:15:44,291 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1810 sec 2024-12-07T15:15:44,295 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 2.1900 sec 2024-12-07T15:15:44,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on 54b89cbce0d0fe96d7c51211398eae76 2024-12-07T15:15:44,566 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 54b89cbce0d0fe96d7c51211398eae76 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-07T15:15:44,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b89cbce0d0fe96d7c51211398eae76, store=A 2024-12-07T15:15:44,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:15:44,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b89cbce0d0fe96d7c51211398eae76, store=B 2024-12-07T15:15:44,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:15:44,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b89cbce0d0fe96d7c51211398eae76, store=C 2024-12-07T15:15:44,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:15:44,571 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/A/deb6b4d4c46b4c65baa2ebb7da18a213 is 50, key is test_row_0/A:col10/1733584544564/Put/seqid=0 2024-12-07T15:15:44,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742289_1465 (size=14341) 2024-12-07T15:15:44,575 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/A/deb6b4d4c46b4c65baa2ebb7da18a213 2024-12-07T15:15:44,589 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/B/cb76d07c6e9d465aaf6127771dfa97c9 is 50, key is test_row_0/B:col10/1733584544564/Put/seqid=0 2024-12-07T15:15:44,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742290_1466 (size=12001) 2024-12-07T15:15:44,624 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:44,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43952 deadline: 1733584604612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:44,624 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:44,624 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:44,624 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:44,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:44,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43912 deadline: 1733584604613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:44,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:44,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:44,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43928 deadline: 1733584604614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:44,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:44,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:44,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:44,627 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:44,627 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:44,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43956 deadline: 1733584604615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:44,627 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:44,627 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:44,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:44,627 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:44,627 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:44,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1733584604622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:44,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:44,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:44,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:44,743 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:44,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43952 deadline: 1733584604733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:44,743 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:44,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43912 deadline: 1733584604733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:44,743 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:44,743 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:44,743 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:44,743 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:44,743 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:44,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43928 deadline: 1733584604733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:44,743 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:44,743 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:44,744 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:44,744 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:44,744 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:44,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43956 deadline: 1733584604733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:44,744 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:44,744 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:44,744 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:44,744 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:44,744 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:44,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1733584604733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:44,744 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:44,744 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:44,744 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:44,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:44,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43952 deadline: 1733584604953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:44,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:44,964 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:44,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43912 deadline: 1733584604953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:44,964 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:44,964 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:44,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:44,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:44,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43928 deadline: 1733584604953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:44,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:44,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:44,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:44,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:44,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:44,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:44,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43956 deadline: 1733584604953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:44,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:44,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1733584604953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:44,965 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:44,965 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:44,965 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:44,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:44,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:44,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:44,993 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/B/cb76d07c6e9d465aaf6127771dfa97c9 2024-12-07T15:15:45,000 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/C/6d440737de324e1690c47da1ef80d2e7 is 50, key is test_row_0/C:col10/1733584544564/Put/seqid=0 2024-12-07T15:15:45,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742291_1467 (size=12001) 2024-12-07T15:15:45,006 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/C/6d440737de324e1690c47da1ef80d2e7 2024-12-07T15:15:45,010 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/A/deb6b4d4c46b4c65baa2ebb7da18a213 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/deb6b4d4c46b4c65baa2ebb7da18a213 2024-12-07T15:15:45,013 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/deb6b4d4c46b4c65baa2ebb7da18a213, entries=200, sequenceid=118, filesize=14.0 K 2024-12-07T15:15:45,013 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/B/cb76d07c6e9d465aaf6127771dfa97c9 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/cb76d07c6e9d465aaf6127771dfa97c9 2024-12-07T15:15:45,017 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/cb76d07c6e9d465aaf6127771dfa97c9, entries=150, sequenceid=118, filesize=11.7 K 2024-12-07T15:15:45,017 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/C/6d440737de324e1690c47da1ef80d2e7 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/6d440737de324e1690c47da1ef80d2e7 2024-12-07T15:15:45,023 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/6d440737de324e1690c47da1ef80d2e7, entries=150, sequenceid=118, filesize=11.7 K 2024-12-07T15:15:45,024 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 54b89cbce0d0fe96d7c51211398eae76 in 458ms, sequenceid=118, compaction requested=true 2024-12-07T15:15:45,024 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 54b89cbce0d0fe96d7c51211398eae76: 2024-12-07T15:15:45,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54b89cbce0d0fe96d7c51211398eae76:A, priority=-2147483648, current under compaction store size is 1 2024-12-07T15:15:45,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:15:45,024 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:15:45,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54b89cbce0d0fe96d7c51211398eae76:B, priority=-2147483648, current under compaction store size is 2 2024-12-07T15:15:45,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:15:45,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54b89cbce0d0fe96d7c51211398eae76:C, priority=-2147483648, current under compaction store size is 3 2024-12-07T15:15:45,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T15:15:45,024 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:15:45,025 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38549 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:15:45,025 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1541): 54b89cbce0d0fe96d7c51211398eae76/A is initiating minor compaction (all files) 2024-12-07T15:15:45,025 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:15:45,025 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54b89cbce0d0fe96d7c51211398eae76/A in TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:45,025 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1541): 54b89cbce0d0fe96d7c51211398eae76/B is initiating minor compaction (all files) 2024-12-07T15:15:45,025 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54b89cbce0d0fe96d7c51211398eae76/B in TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:45,025 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/6c20ccda6d184c62b3ffe21d39b56d97, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/a5cf5ec83742473594991e6ed0e89f4b, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/deb6b4d4c46b4c65baa2ebb7da18a213] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp, totalSize=37.6 K 2024-12-07T15:15:45,025 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/ff8575e113df49d89e98792faf4f01aa, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/ccb1ba5153064582ad94672f1b720ef1, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/cb76d07c6e9d465aaf6127771dfa97c9] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp, totalSize=35.4 K 2024-12-07T15:15:45,026 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting ff8575e113df49d89e98792faf4f01aa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1733584541547 2024-12-07T15:15:45,026 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c20ccda6d184c62b3ffe21d39b56d97, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1733584541547 2024-12-07T15:15:45,026 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting ccb1ba5153064582ad94672f1b720ef1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1733584541701 2024-12-07T15:15:45,026 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting a5cf5ec83742473594991e6ed0e89f4b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1733584541701 2024-12-07T15:15:45,026 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting cb76d07c6e9d465aaf6127771dfa97c9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1733584543931 2024-12-07T15:15:45,026 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting deb6b4d4c46b4c65baa2ebb7da18a213, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1733584543930 2024-12-07T15:15:45,034 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54b89cbce0d0fe96d7c51211398eae76#A#compaction#394 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:15:45,035 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/A/bdccd8c0a164446ba1c2785f6bfc9aa6 is 50, key is test_row_0/A:col10/1733584544564/Put/seqid=0 2024-12-07T15:15:45,035 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54b89cbce0d0fe96d7c51211398eae76#B#compaction#395 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:15:45,035 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/B/9392f77ed7bf40afacb77e32371616a1 is 50, key is test_row_0/B:col10/1733584544564/Put/seqid=0 2024-12-07T15:15:45,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742292_1468 (size=12309) 2024-12-07T15:15:45,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742293_1469 (size=12309) 2024-12-07T15:15:45,046 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/B/9392f77ed7bf40afacb77e32371616a1 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/9392f77ed7bf40afacb77e32371616a1 2024-12-07T15:15:45,051 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54b89cbce0d0fe96d7c51211398eae76/B of 54b89cbce0d0fe96d7c51211398eae76 into 9392f77ed7bf40afacb77e32371616a1(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:15:45,051 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54b89cbce0d0fe96d7c51211398eae76: 2024-12-07T15:15:45,051 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., storeName=54b89cbce0d0fe96d7c51211398eae76/B, priority=13, startTime=1733584545024; duration=0sec 2024-12-07T15:15:45,051 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T15:15:45,051 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54b89cbce0d0fe96d7c51211398eae76:B 2024-12-07T15:15:45,051 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:15:45,052 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:15:45,052 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1541): 54b89cbce0d0fe96d7c51211398eae76/C is initiating minor compaction (all files) 2024-12-07T15:15:45,052 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54b89cbce0d0fe96d7c51211398eae76/C in TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:45,052 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/de53b8864ad944a1bb39c0f7830cf6d0, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/3a06ff06cc7c4b5c81e43799748b6a7c, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/6d440737de324e1690c47da1ef80d2e7] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp, totalSize=35.4 K 2024-12-07T15:15:45,052 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting de53b8864ad944a1bb39c0f7830cf6d0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1733584541547 2024-12-07T15:15:45,053 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a06ff06cc7c4b5c81e43799748b6a7c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1733584541701 2024-12-07T15:15:45,053 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d440737de324e1690c47da1ef80d2e7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1733584543931 2024-12-07T15:15:45,062 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54b89cbce0d0fe96d7c51211398eae76#C#compaction#396 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:15:45,062 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/C/c3c5f1947f2d4cd9ba452be91a792b3f is 50, key is test_row_0/C:col10/1733584544564/Put/seqid=0 2024-12-07T15:15:45,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742294_1470 (size=12309) 2024-12-07T15:15:45,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on 54b89cbce0d0fe96d7c51211398eae76 2024-12-07T15:15:45,275 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 54b89cbce0d0fe96d7c51211398eae76 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-07T15:15:45,275 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b89cbce0d0fe96d7c51211398eae76, store=A 2024-12-07T15:15:45,275 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:15:45,275 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b89cbce0d0fe96d7c51211398eae76, store=B 2024-12-07T15:15:45,275 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:15:45,275 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b89cbce0d0fe96d7c51211398eae76, store=C 2024-12-07T15:15:45,275 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:15:45,280 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/A/1b6d311ed3964ec9a446855f29699236 is 50, key is test_row_0/A:col10/1733584545273/Put/seqid=0 2024-12-07T15:15:45,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742295_1471 (size=14541) 2024-12-07T15:15:45,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:45,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43952 deadline: 1733584605291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:45,320 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:45,321 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:45,321 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:45,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:45,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43912 deadline: 1733584605319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:45,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:45,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:45,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:45,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:45,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43956 deadline: 1733584605319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:45,336 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:45,336 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:45,336 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:45,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:45,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43928 deadline: 1733584605319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:45,336 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:45,336 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:45,336 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:45,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:45,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1733584605320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:45,337 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:45,337 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:45,337 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:45,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:45,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43952 deadline: 1733584605424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:45,431 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:45,431 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:45,431 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:45,447 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/A/bdccd8c0a164446ba1c2785f6bfc9aa6 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/bdccd8c0a164446ba1c2785f6bfc9aa6 2024-12-07T15:15:45,451 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54b89cbce0d0fe96d7c51211398eae76/A of 54b89cbce0d0fe96d7c51211398eae76 into bdccd8c0a164446ba1c2785f6bfc9aa6(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:15:45,451 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54b89cbce0d0fe96d7c51211398eae76: 2024-12-07T15:15:45,451 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., storeName=54b89cbce0d0fe96d7c51211398eae76/A, priority=13, startTime=1733584545024; duration=0sec 2024-12-07T15:15:45,451 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:15:45,451 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54b89cbce0d0fe96d7c51211398eae76:A 2024-12-07T15:15:45,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:45,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43912 deadline: 1733584605443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:45,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:45,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43956 deadline: 1733584605443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:45,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:45,454 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:45,454 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:45,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:45,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43928 deadline: 1733584605443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:45,454 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:45,454 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:45,454 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:45,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:45,454 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:45,454 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:45,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1733584605443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:45,454 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:45,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:45,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:45,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:45,469 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/C/c3c5f1947f2d4cd9ba452be91a792b3f as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/c3c5f1947f2d4cd9ba452be91a792b3f 2024-12-07T15:15:45,473 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54b89cbce0d0fe96d7c51211398eae76/C of 54b89cbce0d0fe96d7c51211398eae76 into c3c5f1947f2d4cd9ba452be91a792b3f(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:15:45,473 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54b89cbce0d0fe96d7c51211398eae76: 2024-12-07T15:15:45,473 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., storeName=54b89cbce0d0fe96d7c51211398eae76/C, priority=13, startTime=1733584545024; duration=0sec 2024-12-07T15:15:45,473 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:15:45,473 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54b89cbce0d0fe96d7c51211398eae76:C 2024-12-07T15:15:45,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:45,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43952 deadline: 1733584605643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:45,651 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:45,651 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:45,651 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:45,669 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:45,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43912 deadline: 1733584605663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:45,670 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:45,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:45,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43956 deadline: 1733584605663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:45,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:45,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:45,670 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:45,670 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:45,670 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:45,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43928 deadline: 1733584605663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:45,670 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:45,670 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:45,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:45,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:45,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1733584605663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:45,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:45,671 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:45,671 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:45,671 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:45,684 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/A/1b6d311ed3964ec9a446855f29699236 2024-12-07T15:15:45,691 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/B/dd8eb0296fe64e81997a690a0d0f93fd is 50, key is test_row_0/B:col10/1733584545273/Put/seqid=0 2024-12-07T15:15:45,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742296_1472 (size=12151) 2024-12-07T15:15:45,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:45,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43952 deadline: 1733584605968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:45,974 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:45,974 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:45,974 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:45,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:45,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1733584605973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:45,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:45,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43912 deadline: 1733584605973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:45,979 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:45,979 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:45,979 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:45,980 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:45,980 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:45,980 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:45,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:45,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43956 deadline: 1733584605983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:45,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:45,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43928 deadline: 1733584605983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:45,991 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:45,991 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:45,991 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:45,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:45,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:45,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:46,098 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/B/dd8eb0296fe64e81997a690a0d0f93fd 2024-12-07T15:15:46,105 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/C/e66c14fa49e74bd399ab01cffb0e1e0f is 50, key is test_row_0/C:col10/1733584545273/Put/seqid=0 2024-12-07T15:15:46,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742297_1473 (size=12151) 2024-12-07T15:15:46,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-07T15:15:46,263 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T15:15:46,264 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T15:15:46,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-12-07T15:15:46,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-07T15:15:46,265 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T15:15:46,266 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T15:15:46,266 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T15:15:46,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-07T15:15:46,418 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-07T15:15:46,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:46,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. as already flushing 2024-12-07T15:15:46,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:46,418 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:46,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:46,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:46,493 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:46,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43952 deadline: 1733584606484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:46,494 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:46,494 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:46,494 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:46,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:46,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1733584606493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:46,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:46,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43912 deadline: 1733584606493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:46,497 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:46,497 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:46,497 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:46,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:46,497 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:46,497 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:46,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43928 deadline: 1733584606493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:46,497 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:46,497 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:46,497 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:46,497 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:46,509 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/C/e66c14fa49e74bd399ab01cffb0e1e0f 2024-12-07T15:15:46,509 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:46,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43956 deadline: 1733584606503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:46,510 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:46,510 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:46,510 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:46,513 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/A/1b6d311ed3964ec9a446855f29699236 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/1b6d311ed3964ec9a446855f29699236 2024-12-07T15:15:46,516 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/1b6d311ed3964ec9a446855f29699236, entries=200, sequenceid=139, filesize=14.2 K 2024-12-07T15:15:46,517 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/B/dd8eb0296fe64e81997a690a0d0f93fd as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/dd8eb0296fe64e81997a690a0d0f93fd 2024-12-07T15:15:46,521 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/dd8eb0296fe64e81997a690a0d0f93fd, entries=150, sequenceid=139, filesize=11.9 K 2024-12-07T15:15:46,522 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/C/e66c14fa49e74bd399ab01cffb0e1e0f as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/e66c14fa49e74bd399ab01cffb0e1e0f 2024-12-07T15:15:46,526 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/e66c14fa49e74bd399ab01cffb0e1e0f, entries=150, sequenceid=139, filesize=11.9 K 2024-12-07T15:15:46,526 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 54b89cbce0d0fe96d7c51211398eae76 in 1251ms, sequenceid=139, compaction requested=false 2024-12-07T15:15:46,527 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 54b89cbce0d0fe96d7c51211398eae76: 2024-12-07T15:15:46,571 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-07T15:15:46,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:46,572 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing 54b89cbce0d0fe96d7c51211398eae76 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-07T15:15:46,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b89cbce0d0fe96d7c51211398eae76, store=A 2024-12-07T15:15:46,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:15:46,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b89cbce0d0fe96d7c51211398eae76, store=B 2024-12-07T15:15:46,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:15:46,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b89cbce0d0fe96d7c51211398eae76, store=C 2024-12-07T15:15:46,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:15:46,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/A/2e46a5a69b754281ba17e10078630f06 is 50, key is test_row_0/A:col10/1733584545290/Put/seqid=0 2024-12-07T15:15:46,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-07T15:15:46,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742298_1474 (size=12151) 2024-12-07T15:15:46,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-07T15:15:46,988 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/A/2e46a5a69b754281ba17e10078630f06 2024-12-07T15:15:47,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/B/61f305e26f974823bf5df076dbd6dc51 is 50, key is test_row_0/B:col10/1733584545290/Put/seqid=0 2024-12-07T15:15:47,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742299_1475 (size=12151) 2024-12-07T15:15:47,058 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/B/61f305e26f974823bf5df076dbd6dc51 2024-12-07T15:15:47,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/C/7f9d90f6e337469d952001046a6d38ee is 50, key is test_row_0/C:col10/1733584545290/Put/seqid=0 2024-12-07T15:15:47,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742300_1476 (size=12151) 2024-12-07T15:15:47,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-07T15:15:47,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on 54b89cbce0d0fe96d7c51211398eae76 2024-12-07T15:15:47,507 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. as already flushing 2024-12-07T15:15:47,515 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/C/7f9d90f6e337469d952001046a6d38ee 2024-12-07T15:15:47,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/A/2e46a5a69b754281ba17e10078630f06 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/2e46a5a69b754281ba17e10078630f06 2024-12-07T15:15:47,523 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/2e46a5a69b754281ba17e10078630f06, entries=150, sequenceid=153, filesize=11.9 K 2024-12-07T15:15:47,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/B/61f305e26f974823bf5df076dbd6dc51 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/61f305e26f974823bf5df076dbd6dc51 2024-12-07T15:15:47,528 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/61f305e26f974823bf5df076dbd6dc51, entries=150, sequenceid=153, filesize=11.9 K 2024-12-07T15:15:47,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/C/7f9d90f6e337469d952001046a6d38ee as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/7f9d90f6e337469d952001046a6d38ee 2024-12-07T15:15:47,532 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/7f9d90f6e337469d952001046a6d38ee, entries=150, sequenceid=153, filesize=11.9 K 2024-12-07T15:15:47,561 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=46.96 KB/48090 for 54b89cbce0d0fe96d7c51211398eae76 in 990ms, sequenceid=153, compaction requested=true 2024-12-07T15:15:47,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for 54b89cbce0d0fe96d7c51211398eae76: 2024-12-07T15:15:47,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:47,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on 54b89cbce0d0fe96d7c51211398eae76 2024-12-07T15:15:47,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-12-07T15:15:47,562 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 54b89cbce0d0fe96d7c51211398eae76 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-07T15:15:47,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-12-07T15:15:47,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b89cbce0d0fe96d7c51211398eae76, store=A 2024-12-07T15:15:47,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:15:47,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b89cbce0d0fe96d7c51211398eae76, store=B 2024-12-07T15:15:47,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:15:47,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b89cbce0d0fe96d7c51211398eae76, store=C 2024-12-07T15:15:47,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:15:47,567 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/A/48b2da53004b490a955cde74ae017810 is 50, key is test_row_0/A:col10/1733584547535/Put/seqid=0 2024-12-07T15:15:47,568 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-12-07T15:15:47,568 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2970 sec 2024-12-07T15:15:47,570 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 1.3040 sec 2024-12-07T15:15:47,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742301_1477 (size=12151) 2024-12-07T15:15:47,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:47,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43952 deadline: 1733584607599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:47,615 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:47,615 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:47,615 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:47,615 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:47,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43956 deadline: 1733584607600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:47,615 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:47,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43912 deadline: 1733584607601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:47,615 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:47,615 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:47,615 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:47,615 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:47,615 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:47,615 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:47,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:47,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1733584607613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:47,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:47,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:47,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:47,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:47,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43928 deadline: 1733584607615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:47,629 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:47,629 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:47,629 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:47,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:47,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43952 deadline: 1733584607723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:47,734 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:47,734 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:47,734 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:47,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43956 deadline: 1733584607723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:47,734 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:47,734 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:47,734 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:47,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43912 deadline: 1733584607723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:47,734 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:47,734 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:47,734 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:47,735 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:47,735 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:47,742 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:47,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1733584607733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:47,743 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:47,743 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:47,743 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:47,743 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:47,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43928 deadline: 1733584607733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:47,743 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:47,743 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:47,743 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:47,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:47,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43952 deadline: 1733584607943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:47,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:47,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43956 deadline: 1733584607943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:47,954 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:47,954 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:47,954 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:47,954 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:47,954 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:47,954 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:47,954 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:47,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43912 deadline: 1733584607943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:47,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:47,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:47,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:47,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:47,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1733584607953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:47,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:47,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:47,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43928 deadline: 1733584607954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:47,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:47,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:47,966 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:47,966 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:47,966 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:47,979 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/A/48b2da53004b490a955cde74ae017810 2024-12-07T15:15:47,987 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/B/1b7478547f9c45cdac97dc5fbd7ade80 is 50, key is test_row_0/B:col10/1733584547535/Put/seqid=0 2024-12-07T15:15:47,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742302_1478 (size=12151) 2024-12-07T15:15:48,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:48,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43952 deadline: 1733584608263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:48,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:48,269 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:48,269 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:48,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43956 deadline: 1733584608263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:48,269 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:48,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:48,269 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:48,269 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:48,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43912 deadline: 1733584608263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:48,269 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:48,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:48,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:48,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:48,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:48,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1733584608273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:48,284 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:48,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:48,284 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:48,284 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:48,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43928 deadline: 1733584608273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:48,284 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:48,285 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:48,285 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:48,391 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/B/1b7478547f9c45cdac97dc5fbd7ade80 2024-12-07T15:15:48,397 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/C/e89b5c3bcb7d4d2fb2b5c7c3c8fc5749 is 50, key is test_row_0/C:col10/1733584547535/Put/seqid=0 2024-12-07T15:15:48,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742303_1479 (size=12151) 2024-12-07T15:15:48,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-07T15:15:48,414 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T15:15:48,414 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T15:15:48,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-12-07T15:15:48,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-07T15:15:48,416 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T15:15:48,416 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T15:15:48,416 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T15:15:48,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-07T15:15:48,568 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-07T15:15:48,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:48,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. as already flushing 2024-12-07T15:15:48,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:48,569 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:48,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:48,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:48,721 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-07T15:15:48,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:48,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. as already flushing 2024-12-07T15:15:48,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:48,721 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:48,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:48,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:48,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-07T15:15:48,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:48,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43952 deadline: 1733584608773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:48,782 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:48,782 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:48,782 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:48,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:48,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43956 deadline: 1733584608783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:48,794 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:48,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:48,794 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:48,794 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:48,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43912 deadline: 1733584608783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:48,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:48,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:48,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:48,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:48,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1733584608793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:48,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:48,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:48,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:48,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:48,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43928 deadline: 1733584608793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:48,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:48,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:48,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:48,801 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/C/e89b5c3bcb7d4d2fb2b5c7c3c8fc5749 2024-12-07T15:15:48,805 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/A/48b2da53004b490a955cde74ae017810 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/48b2da53004b490a955cde74ae017810 2024-12-07T15:15:48,808 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/48b2da53004b490a955cde74ae017810, entries=150, sequenceid=166, filesize=11.9 K 2024-12-07T15:15:48,808 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/B/1b7478547f9c45cdac97dc5fbd7ade80 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/1b7478547f9c45cdac97dc5fbd7ade80 2024-12-07T15:15:48,811 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/1b7478547f9c45cdac97dc5fbd7ade80, entries=150, sequenceid=166, filesize=11.9 K 2024-12-07T15:15:48,812 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/C/e89b5c3bcb7d4d2fb2b5c7c3c8fc5749 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/e89b5c3bcb7d4d2fb2b5c7c3c8fc5749 2024-12-07T15:15:48,815 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/e89b5c3bcb7d4d2fb2b5c7c3c8fc5749, entries=150, sequenceid=166, filesize=11.9 K 2024-12-07T15:15:48,816 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 54b89cbce0d0fe96d7c51211398eae76 in 1254ms, sequenceid=166, compaction requested=true 2024-12-07T15:15:48,816 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 54b89cbce0d0fe96d7c51211398eae76: 2024-12-07T15:15:48,816 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-07T15:15:48,816 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54b89cbce0d0fe96d7c51211398eae76:A, priority=-2147483648, current under compaction store size is 1 2024-12-07T15:15:48,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:15:48,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54b89cbce0d0fe96d7c51211398eae76:B, priority=-2147483648, current under compaction store size is 2 2024-12-07T15:15:48,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:15:48,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54b89cbce0d0fe96d7c51211398eae76:C, priority=-2147483648, current under compaction store size is 3 2024-12-07T15:15:48,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T15:15:48,817 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-07T15:15:48,817 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51152 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-07T15:15:48,817 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1541): 54b89cbce0d0fe96d7c51211398eae76/A is initiating minor compaction (all files) 2024-12-07T15:15:48,818 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54b89cbce0d0fe96d7c51211398eae76/A in TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:48,818 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/bdccd8c0a164446ba1c2785f6bfc9aa6, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/1b6d311ed3964ec9a446855f29699236, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/2e46a5a69b754281ba17e10078630f06, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/48b2da53004b490a955cde74ae017810] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp, totalSize=50.0 K 2024-12-07T15:15:48,818 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48762 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-07T15:15:48,818 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting bdccd8c0a164446ba1c2785f6bfc9aa6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1733584543931 2024-12-07T15:15:48,818 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1541): 54b89cbce0d0fe96d7c51211398eae76/B is initiating minor compaction (all files) 2024-12-07T15:15:48,818 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54b89cbce0d0fe96d7c51211398eae76/B in TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:48,818 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/9392f77ed7bf40afacb77e32371616a1, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/dd8eb0296fe64e81997a690a0d0f93fd, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/61f305e26f974823bf5df076dbd6dc51, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/1b7478547f9c45cdac97dc5fbd7ade80] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp, totalSize=47.6 K 2024-12-07T15:15:48,818 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b6d311ed3964ec9a446855f29699236, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1733584544612 2024-12-07T15:15:48,818 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e46a5a69b754281ba17e10078630f06, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1733584545290 2024-12-07T15:15:48,818 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 9392f77ed7bf40afacb77e32371616a1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1733584543931 2024-12-07T15:15:48,819 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48b2da53004b490a955cde74ae017810, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1733584547532 2024-12-07T15:15:48,819 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting dd8eb0296fe64e81997a690a0d0f93fd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1733584544612 2024-12-07T15:15:48,819 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 61f305e26f974823bf5df076dbd6dc51, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1733584545290 2024-12-07T15:15:48,819 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b7478547f9c45cdac97dc5fbd7ade80, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1733584547532 2024-12-07T15:15:48,829 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54b89cbce0d0fe96d7c51211398eae76#A#compaction#407 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:15:48,829 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54b89cbce0d0fe96d7c51211398eae76#B#compaction#406 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:15:48,830 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/B/45d010b62f8b48aeb0612bbb3878976b is 50, key is test_row_0/B:col10/1733584547535/Put/seqid=0 2024-12-07T15:15:48,830 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/A/36616ec865804a70b75a5f6c2d6439da is 50, key is test_row_0/A:col10/1733584547535/Put/seqid=0 2024-12-07T15:15:48,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742304_1480 (size=12595) 2024-12-07T15:15:48,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742305_1481 (size=12595) 2024-12-07T15:15:48,843 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/B/45d010b62f8b48aeb0612bbb3878976b as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/45d010b62f8b48aeb0612bbb3878976b 2024-12-07T15:15:48,847 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 54b89cbce0d0fe96d7c51211398eae76/B of 54b89cbce0d0fe96d7c51211398eae76 into 45d010b62f8b48aeb0612bbb3878976b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:15:48,847 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54b89cbce0d0fe96d7c51211398eae76: 2024-12-07T15:15:48,847 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., storeName=54b89cbce0d0fe96d7c51211398eae76/B, priority=12, startTime=1733584548817; duration=0sec 2024-12-07T15:15:48,847 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T15:15:48,848 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54b89cbce0d0fe96d7c51211398eae76:B 2024-12-07T15:15:48,848 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-07T15:15:48,848 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48762 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-07T15:15:48,849 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1541): 54b89cbce0d0fe96d7c51211398eae76/C is initiating minor compaction (all files) 2024-12-07T15:15:48,849 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54b89cbce0d0fe96d7c51211398eae76/C in TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:48,849 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/c3c5f1947f2d4cd9ba452be91a792b3f, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/e66c14fa49e74bd399ab01cffb0e1e0f, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/7f9d90f6e337469d952001046a6d38ee, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/e89b5c3bcb7d4d2fb2b5c7c3c8fc5749] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp, totalSize=47.6 K 2024-12-07T15:15:48,849 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting c3c5f1947f2d4cd9ba452be91a792b3f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1733584543931 2024-12-07T15:15:48,851 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting e66c14fa49e74bd399ab01cffb0e1e0f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1733584544612 2024-12-07T15:15:48,852 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f9d90f6e337469d952001046a6d38ee, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1733584545290 2024-12-07T15:15:48,852 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting e89b5c3bcb7d4d2fb2b5c7c3c8fc5749, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1733584547532 2024-12-07T15:15:48,858 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54b89cbce0d0fe96d7c51211398eae76#C#compaction#408 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:15:48,858 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/C/6f9d4549067f47b09c526f9732ba914e is 50, key is test_row_0/C:col10/1733584547535/Put/seqid=0 2024-12-07T15:15:48,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742306_1482 (size=12595) 2024-12-07T15:15:48,874 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-07T15:15:48,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:48,874 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing 54b89cbce0d0fe96d7c51211398eae76 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-07T15:15:48,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b89cbce0d0fe96d7c51211398eae76, store=A 2024-12-07T15:15:48,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:15:48,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b89cbce0d0fe96d7c51211398eae76, store=B 2024-12-07T15:15:48,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:15:48,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b89cbce0d0fe96d7c51211398eae76, store=C 2024-12-07T15:15:48,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:15:48,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/A/887696742482454da8a3be6344ec1aea is 50, key is test_row_0/A:col10/1733584547611/Put/seqid=0 2024-12-07T15:15:48,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742307_1483 (size=12151) 2024-12-07T15:15:48,882 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/A/887696742482454da8a3be6344ec1aea 2024-12-07T15:15:48,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/B/d7d5dc8b80924c3d8475be0fd3219987 is 50, key is test_row_0/B:col10/1733584547611/Put/seqid=0 2024-12-07T15:15:48,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742308_1484 (size=12151) 2024-12-07T15:15:49,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-07T15:15:49,242 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/A/36616ec865804a70b75a5f6c2d6439da as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/36616ec865804a70b75a5f6c2d6439da 2024-12-07T15:15:49,246 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 54b89cbce0d0fe96d7c51211398eae76/A of 54b89cbce0d0fe96d7c51211398eae76 into 36616ec865804a70b75a5f6c2d6439da(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:15:49,246 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54b89cbce0d0fe96d7c51211398eae76: 2024-12-07T15:15:49,246 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., storeName=54b89cbce0d0fe96d7c51211398eae76/A, priority=12, startTime=1733584548816; duration=0sec 2024-12-07T15:15:49,246 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:15:49,246 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54b89cbce0d0fe96d7c51211398eae76:A 2024-12-07T15:15:49,270 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/C/6f9d4549067f47b09c526f9732ba914e as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/6f9d4549067f47b09c526f9732ba914e 2024-12-07T15:15:49,274 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 54b89cbce0d0fe96d7c51211398eae76/C of 54b89cbce0d0fe96d7c51211398eae76 into 6f9d4549067f47b09c526f9732ba914e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:15:49,274 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54b89cbce0d0fe96d7c51211398eae76: 2024-12-07T15:15:49,274 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., storeName=54b89cbce0d0fe96d7c51211398eae76/C, priority=12, startTime=1733584548817; duration=0sec 2024-12-07T15:15:49,274 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:15:49,274 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54b89cbce0d0fe96d7c51211398eae76:C 2024-12-07T15:15:49,293 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/B/d7d5dc8b80924c3d8475be0fd3219987 2024-12-07T15:15:49,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/C/b9c3b1ec8fd148909f4062a333b89910 is 50, key is test_row_0/C:col10/1733584547611/Put/seqid=0 2024-12-07T15:15:49,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742309_1485 (size=12151) 2024-12-07T15:15:49,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-07T15:15:49,706 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/C/b9c3b1ec8fd148909f4062a333b89910 2024-12-07T15:15:49,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/A/887696742482454da8a3be6344ec1aea as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/887696742482454da8a3be6344ec1aea 2024-12-07T15:15:49,714 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/887696742482454da8a3be6344ec1aea, entries=150, sequenceid=186, filesize=11.9 K 2024-12-07T15:15:49,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/B/d7d5dc8b80924c3d8475be0fd3219987 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/d7d5dc8b80924c3d8475be0fd3219987 2024-12-07T15:15:49,723 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/d7d5dc8b80924c3d8475be0fd3219987, entries=150, sequenceid=186, filesize=11.9 K 2024-12-07T15:15:49,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/C/b9c3b1ec8fd148909f4062a333b89910 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/b9c3b1ec8fd148909f4062a333b89910 2024-12-07T15:15:49,730 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/b9c3b1ec8fd148909f4062a333b89910, entries=150, sequenceid=186, filesize=11.9 K 2024-12-07T15:15:49,731 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=0 B/0 for 54b89cbce0d0fe96d7c51211398eae76 in 856ms, sequenceid=186, compaction requested=false 2024-12-07T15:15:49,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for 54b89cbce0d0fe96d7c51211398eae76: 2024-12-07T15:15:49,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:49,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-12-07T15:15:49,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-12-07T15:15:49,734 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-12-07T15:15:49,734 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3160 sec 2024-12-07T15:15:49,736 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 1.3210 sec 2024-12-07T15:15:49,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on 54b89cbce0d0fe96d7c51211398eae76 2024-12-07T15:15:49,808 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 54b89cbce0d0fe96d7c51211398eae76 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-07T15:15:49,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b89cbce0d0fe96d7c51211398eae76, store=A 2024-12-07T15:15:49,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:15:49,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b89cbce0d0fe96d7c51211398eae76, store=B 2024-12-07T15:15:49,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:15:49,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b89cbce0d0fe96d7c51211398eae76, store=C 2024-12-07T15:15:49,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:15:49,814 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/A/38f20619c1954ee9b29909eb2509766b is 50, key is test_row_0/A:col10/1733584549806/Put/seqid=0 2024-12-07T15:15:49,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742310_1486 (size=14541) 2024-12-07T15:15:49,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:49,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43928 deadline: 1733584609842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:49,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:49,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43956 deadline: 1733584609843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:49,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:49,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1733584609845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:49,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:49,857 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1733584549824, max timestamp=1733584549854Num uniques 859; 2024-12-07T15:15:49,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43912 deadline: 1733584609854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:49,858 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-12-07T15:15:49,858 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1733584549824, max timestamp=1733584549854Num uniques 859; 2024-12-07T15:15:49,858 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1733584549824, max timestamp=1733584549854Num uniques 859; 2024-12-07T15:15:49,858 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-12-07T15:15:49,858 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-12-07T15:15:49,858 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:49,858 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:49,858 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:49,858 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:49,858 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:49,858 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:49,858 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:49,858 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:49,858 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:49,858 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:49,858 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:49,858 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:49,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:49,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43952 deadline: 1733584609951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:49,963 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:49,963 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:49,963 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:49,972 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:49,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1733584609963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:49,972 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:49,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:49,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:49,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43928 deadline: 1733584609963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:49,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:49,972 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:49,973 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:49,973 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:49,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43956 deadline: 1733584609963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:49,973 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:49,973 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:49,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:49,973 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:49,973 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:49,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43912 deadline: 1733584609963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:49,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:49,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:49,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:50,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:50,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43952 deadline: 1733584610073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:50,077 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:50,077 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:50,077 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:50,191 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:50,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1733584610183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:50,191 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:50,192 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:50,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43928 deadline: 1733584610183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:50,192 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:50,192 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:50,192 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:50,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:50,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:50,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:50,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43956 deadline: 1733584610183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:50,192 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:50,192 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:50,192 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:50,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43912 deadline: 1733584610183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:50,192 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:50,192 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:50,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:50,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:50,223 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/A/38f20619c1954ee9b29909eb2509766b 2024-12-07T15:15:50,229 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/B/7b3ac480e77540de94e578a3ed0a6a07 is 50, key is test_row_0/B:col10/1733584549806/Put/seqid=0 2024-12-07T15:15:50,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742311_1487 (size=12151) 2024-12-07T15:15:50,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:50,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43952 deadline: 1733584610283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:50,295 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:50,295 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:50,295 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:50,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:50,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43928 deadline: 1733584610493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:50,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:50,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:50,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:50,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:50,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1733584610503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:50,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:50,513 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:50,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43956 deadline: 1733584610503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:50,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:50,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:50,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:50,514 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:50,514 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:50,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43912 deadline: 1733584610503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:50,514 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:50,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:50,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:50,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:50,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-07T15:15:50,563 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T15:15:50,564 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T15:15:50,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-12-07T15:15:50,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-07T15:15:50,565 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T15:15:50,566 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T15:15:50,566 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T15:15:50,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:50,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43952 deadline: 1733584610603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:50,616 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:50,616 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:50,616 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:50,633 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/B/7b3ac480e77540de94e578a3ed0a6a07 2024-12-07T15:15:50,640 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/C/fb6b6c32d8934e7da62aab69e7f6ba5f is 50, key is test_row_0/C:col10/1733584549806/Put/seqid=0 2024-12-07T15:15:50,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742312_1488 (size=12151) 2024-12-07T15:15:50,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-07T15:15:50,718 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-07T15:15:50,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:50,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. as already flushing 2024-12-07T15:15:50,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:50,719 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:50,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:50,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:50,871 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-07T15:15:50,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:50,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. as already flushing 2024-12-07T15:15:50,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:50,872 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:50,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:50,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:50,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-07T15:15:51,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:51,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43928 deadline: 1733584611014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:51,022 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:51,022 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:51,022 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:51,024 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-07T15:15:51,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:51,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. as already flushing 2024-12-07T15:15:51,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:51,024 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:51,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:51,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:51,033 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:51,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1733584611023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:51,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:51,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43956 deadline: 1733584611023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:51,034 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:51,034 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:51,034 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:51,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:51,034 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:51,034 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:51,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43912 deadline: 1733584611023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:51,034 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:51,034 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:51,034 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:51,034 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:51,044 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/C/fb6b6c32d8934e7da62aab69e7f6ba5f 2024-12-07T15:15:51,048 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/A/38f20619c1954ee9b29909eb2509766b as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/38f20619c1954ee9b29909eb2509766b 2024-12-07T15:15:51,052 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/38f20619c1954ee9b29909eb2509766b, entries=200, sequenceid=199, filesize=14.2 K 2024-12-07T15:15:51,052 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/B/7b3ac480e77540de94e578a3ed0a6a07 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/7b3ac480e77540de94e578a3ed0a6a07 2024-12-07T15:15:51,056 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/7b3ac480e77540de94e578a3ed0a6a07, entries=150, sequenceid=199, filesize=11.9 K 2024-12-07T15:15:51,056 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/C/fb6b6c32d8934e7da62aab69e7f6ba5f as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/fb6b6c32d8934e7da62aab69e7f6ba5f 2024-12-07T15:15:51,060 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/fb6b6c32d8934e7da62aab69e7f6ba5f, entries=150, sequenceid=199, filesize=11.9 K 2024-12-07T15:15:51,060 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 54b89cbce0d0fe96d7c51211398eae76 in 1252ms, sequenceid=199, compaction requested=true 2024-12-07T15:15:51,060 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 54b89cbce0d0fe96d7c51211398eae76: 2024-12-07T15:15:51,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54b89cbce0d0fe96d7c51211398eae76:A, priority=-2147483648, current under compaction store size is 1 2024-12-07T15:15:51,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:15:51,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54b89cbce0d0fe96d7c51211398eae76:B, priority=-2147483648, current under compaction store size is 2 2024-12-07T15:15:51,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:15:51,061 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:15:51,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54b89cbce0d0fe96d7c51211398eae76:C, priority=-2147483648, current under compaction store size is 3 2024-12-07T15:15:51,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T15:15:51,061 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:15:51,061 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39287 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:15:51,061 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1541): 54b89cbce0d0fe96d7c51211398eae76/A is initiating minor compaction (all files) 2024-12-07T15:15:51,062 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54b89cbce0d0fe96d7c51211398eae76/A in TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:51,062 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/36616ec865804a70b75a5f6c2d6439da, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/887696742482454da8a3be6344ec1aea, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/38f20619c1954ee9b29909eb2509766b] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp, totalSize=38.4 K 2024-12-07T15:15:51,062 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:15:51,062 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1541): 54b89cbce0d0fe96d7c51211398eae76/B is initiating minor compaction (all files) 2024-12-07T15:15:51,062 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54b89cbce0d0fe96d7c51211398eae76/B in TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:51,062 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/45d010b62f8b48aeb0612bbb3878976b, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/d7d5dc8b80924c3d8475be0fd3219987, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/7b3ac480e77540de94e578a3ed0a6a07] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp, totalSize=36.0 K 2024-12-07T15:15:51,062 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 36616ec865804a70b75a5f6c2d6439da, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1733584547532 2024-12-07T15:15:51,062 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 45d010b62f8b48aeb0612bbb3878976b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1733584547532 2024-12-07T15:15:51,063 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 887696742482454da8a3be6344ec1aea, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1733584547598 2024-12-07T15:15:51,063 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting d7d5dc8b80924c3d8475be0fd3219987, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1733584547598 2024-12-07T15:15:51,063 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 38f20619c1954ee9b29909eb2509766b, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1733584549802 2024-12-07T15:15:51,063 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b3ac480e77540de94e578a3ed0a6a07, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1733584549802 2024-12-07T15:15:51,071 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54b89cbce0d0fe96d7c51211398eae76#A#compaction#415 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:15:51,071 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54b89cbce0d0fe96d7c51211398eae76#B#compaction#416 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:15:51,071 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/A/ec560318fd3c4a16bd05cef580798755 is 50, key is test_row_0/A:col10/1733584549806/Put/seqid=0 2024-12-07T15:15:51,072 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/B/e263ea4699c6403682fc9f5f798b6e27 is 50, key is test_row_0/B:col10/1733584549806/Put/seqid=0 2024-12-07T15:15:51,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742313_1489 (size=12697) 2024-12-07T15:15:51,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742314_1490 (size=12697) 2024-12-07T15:15:51,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on 54b89cbce0d0fe96d7c51211398eae76 2024-12-07T15:15:51,133 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 54b89cbce0d0fe96d7c51211398eae76 3/3 column families, dataSize=140.89 KB heapSize=381.47 KB 2024-12-07T15:15:51,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b89cbce0d0fe96d7c51211398eae76, store=A 2024-12-07T15:15:51,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T15:15:51,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b89cbce0d0fe96d7c51211398eae76, store=B 2024-12-07T15:15:51,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T15:15:51,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b89cbce0d0fe96d7c51211398eae76, store=C 2024-12-07T15:15:51,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T15:15:51,138 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/A/ad3cd1aaf0f04fed8470bbb5cd364179 is 50, key is test_row_0/A:col10/1733584551132/Put/seqid=0 2024-12-07T15:15:51,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742315_1491 (size=14541) 2024-12-07T15:15:51,177 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-07T15:15:51,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:51,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. as already flushing 2024-12-07T15:15:51,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:51,177 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:51,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:51,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:51,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-07T15:15:51,208 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:51,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43952 deadline: 1733584611206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:51,209 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:51,209 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:51,209 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:51,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:51,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43952 deadline: 1733584611314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:51,321 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:51,321 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:51,321 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:51,329 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-07T15:15:51,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:51,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. as already flushing 2024-12-07T15:15:51,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:51,330 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:51,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:51,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:51,481 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/A/ec560318fd3c4a16bd05cef580798755 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/ec560318fd3c4a16bd05cef580798755 2024-12-07T15:15:51,482 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-07T15:15:51,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:51,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. as already flushing 2024-12-07T15:15:51,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:51,482 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:51,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:51,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:51,485 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54b89cbce0d0fe96d7c51211398eae76/A of 54b89cbce0d0fe96d7c51211398eae76 into ec560318fd3c4a16bd05cef580798755(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:15:51,485 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54b89cbce0d0fe96d7c51211398eae76: 2024-12-07T15:15:51,485 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., storeName=54b89cbce0d0fe96d7c51211398eae76/A, priority=13, startTime=1733584551060; duration=0sec 2024-12-07T15:15:51,485 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T15:15:51,485 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54b89cbce0d0fe96d7c51211398eae76:A 2024-12-07T15:15:51,486 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:15:51,486 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:15:51,486 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1541): 54b89cbce0d0fe96d7c51211398eae76/C is initiating minor compaction (all files) 2024-12-07T15:15:51,487 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54b89cbce0d0fe96d7c51211398eae76/C in TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:51,487 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/B/e263ea4699c6403682fc9f5f798b6e27 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/e263ea4699c6403682fc9f5f798b6e27 2024-12-07T15:15:51,487 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/6f9d4549067f47b09c526f9732ba914e, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/b9c3b1ec8fd148909f4062a333b89910, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/fb6b6c32d8934e7da62aab69e7f6ba5f] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp, totalSize=36.0 K 2024-12-07T15:15:51,487 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f9d4549067f47b09c526f9732ba914e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1733584547532 2024-12-07T15:15:51,487 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting b9c3b1ec8fd148909f4062a333b89910, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1733584547598 2024-12-07T15:15:51,488 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb6b6c32d8934e7da62aab69e7f6ba5f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1733584549802 2024-12-07T15:15:51,491 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54b89cbce0d0fe96d7c51211398eae76/B of 54b89cbce0d0fe96d7c51211398eae76 into e263ea4699c6403682fc9f5f798b6e27(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:15:51,491 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54b89cbce0d0fe96d7c51211398eae76: 2024-12-07T15:15:51,491 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., storeName=54b89cbce0d0fe96d7c51211398eae76/B, priority=13, startTime=1733584551061; duration=0sec 2024-12-07T15:15:51,491 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:15:51,491 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54b89cbce0d0fe96d7c51211398eae76:B 2024-12-07T15:15:51,495 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54b89cbce0d0fe96d7c51211398eae76#C#compaction#418 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:15:51,495 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/C/e4afea5931d646f79c50a1ca325ce115 is 50, key is test_row_0/C:col10/1733584549806/Put/seqid=0 2024-12-07T15:15:51,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742316_1492 (size=12697) 2024-12-07T15:15:51,502 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/C/e4afea5931d646f79c50a1ca325ce115 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/e4afea5931d646f79c50a1ca325ce115 2024-12-07T15:15:51,506 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54b89cbce0d0fe96d7c51211398eae76/C of 54b89cbce0d0fe96d7c51211398eae76 into e4afea5931d646f79c50a1ca325ce115(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:15:51,506 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54b89cbce0d0fe96d7c51211398eae76: 2024-12-07T15:15:51,506 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., storeName=54b89cbce0d0fe96d7c51211398eae76/C, priority=13, startTime=1733584551061; duration=0sec 2024-12-07T15:15:51,506 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:15:51,506 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54b89cbce0d0fe96d7c51211398eae76:C 2024-12-07T15:15:51,541 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:51,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43952 deadline: 1733584611533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:51,541 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:51,542 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:51,542 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:51,542 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/A/ad3cd1aaf0f04fed8470bbb5cd364179 2024-12-07T15:15:51,549 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/B/7f9b7eba129a47acbb98f9b7401c3539 is 50, key is test_row_0/B:col10/1733584551132/Put/seqid=0 2024-12-07T15:15:51,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742317_1493 (size=12151) 2024-12-07T15:15:51,635 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-07T15:15:51,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:51,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. as already flushing 2024-12-07T15:15:51,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:51,635 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:51,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:51,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:51,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-07T15:15:51,788 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-07T15:15:51,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:51,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. as already flushing 2024-12-07T15:15:51,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:51,788 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:51,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:51,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:51,860 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:51,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43952 deadline: 1733584611854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:51,861 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:51,861 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:51,861 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:51,941 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-07T15:15:51,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:51,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. as already flushing 2024-12-07T15:15:51,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:51,941 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:51,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:51,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:51,953 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/B/7f9b7eba129a47acbb98f9b7401c3539 2024-12-07T15:15:51,960 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/C/34831ea61f9f4084808d416b314b6ad1 is 50, key is test_row_0/C:col10/1733584551132/Put/seqid=0 2024-12-07T15:15:51,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742318_1494 (size=12151) 2024-12-07T15:15:51,970 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/C/34831ea61f9f4084808d416b314b6ad1 2024-12-07T15:15:51,976 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/A/ad3cd1aaf0f04fed8470bbb5cd364179 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/ad3cd1aaf0f04fed8470bbb5cd364179 2024-12-07T15:15:51,981 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/ad3cd1aaf0f04fed8470bbb5cd364179, entries=200, sequenceid=232, filesize=14.2 K 2024-12-07T15:15:51,982 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/B/7f9b7eba129a47acbb98f9b7401c3539 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/7f9b7eba129a47acbb98f9b7401c3539 2024-12-07T15:15:51,984 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/7f9b7eba129a47acbb98f9b7401c3539, entries=150, sequenceid=232, filesize=11.9 K 2024-12-07T15:15:51,985 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/C/34831ea61f9f4084808d416b314b6ad1 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/34831ea61f9f4084808d416b314b6ad1 2024-12-07T15:15:51,990 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/34831ea61f9f4084808d416b314b6ad1, entries=150, sequenceid=232, filesize=11.9 K 2024-12-07T15:15:51,991 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~381.42 KB/390576, currentSize=46.96 KB/48090 for 54b89cbce0d0fe96d7c51211398eae76 in 858ms, sequenceid=232, compaction requested=false 2024-12-07T15:15:51,991 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 54b89cbce0d0fe96d7c51211398eae76: 2024-12-07T15:15:52,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on 54b89cbce0d0fe96d7c51211398eae76 2024-12-07T15:15:52,036 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 54b89cbce0d0fe96d7c51211398eae76 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-07T15:15:52,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b89cbce0d0fe96d7c51211398eae76, store=A 2024-12-07T15:15:52,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:15:52,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b89cbce0d0fe96d7c51211398eae76, store=B 2024-12-07T15:15:52,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:15:52,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b89cbce0d0fe96d7c51211398eae76, store=C 2024-12-07T15:15:52,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:15:52,042 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/A/c2c30357eec34a4db091e67c5afdf270 is 50, key is test_row_0/A:col10/1733584552034/Put/seqid=0 2024-12-07T15:15:52,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742319_1495 (size=14541) 2024-12-07T15:15:52,046 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/A/c2c30357eec34a4db091e67c5afdf270 2024-12-07T15:15:52,052 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/B/5f676a18ea6b4aedb984bd8d1ec08e1d is 50, key is test_row_0/B:col10/1733584552034/Put/seqid=0 2024-12-07T15:15:52,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742320_1496 (size=12151) 2024-12-07T15:15:52,094 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-07T15:15:52,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:52,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. as already flushing 2024-12-07T15:15:52,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:52,094 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:52,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:52,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:52,124 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1733584552101, max timestamp=1733584552121Num uniques 859; 2024-12-07T15:15:52,124 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-12-07T15:15:52,125 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1733584552101, max timestamp=1733584552121Num uniques 859; 2024-12-07T15:15:52,125 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-12-07T15:15:52,125 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1733584552101, max timestamp=1733584552121Num uniques 859; 2024-12-07T15:15:52,125 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-12-07T15:15:52,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:52,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43912 deadline: 1733584612175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:52,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:52,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:52,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:52,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:52,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:52,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43928 deadline: 1733584612177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:52,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1733584612176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:52,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:52,192 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:52,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:52,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43956 deadline: 1733584612179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:52,192 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:52,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:52,192 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:52,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:52,192 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:52,192 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:52,193 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:52,247 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-07T15:15:52,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:52,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. as already flushing 2024-12-07T15:15:52,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:52,247 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:52,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:52,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:52,288 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:52,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43912 deadline: 1733584612283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:52,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:52,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:52,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:52,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:52,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1733584612303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:52,316 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:52,316 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:52,316 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:52,316 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:52,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43928 deadline: 1733584612303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:52,316 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:52,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:52,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:52,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43956 deadline: 1733584612303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:52,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:52,317 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:52,317 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:52,317 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:52,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:52,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43952 deadline: 1733584612373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:52,381 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:52,381 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:52,381 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:52,399 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-07T15:15:52,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:52,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. as already flushing 2024-12-07T15:15:52,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:52,400 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:52,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:52,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:52,458 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/B/5f676a18ea6b4aedb984bd8d1ec08e1d 2024-12-07T15:15:52,464 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/C/95e2a3008942455b9f9c04ca585ddf4d is 50, key is test_row_0/C:col10/1733584552034/Put/seqid=0 2024-12-07T15:15:52,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742321_1497 (size=12151) 2024-12-07T15:15:52,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:52,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43912 deadline: 1733584612493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:52,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:52,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:52,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:52,533 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:52,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1733584612523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:52,534 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:52,534 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:52,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43928 deadline: 1733584612523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:52,534 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:52,534 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:52,534 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:52,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43956 deadline: 1733584612523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:52,534 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:52,534 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:52,534 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:52,534 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:52,534 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:52,534 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:52,552 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-07T15:15:52,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:52,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. as already flushing 2024-12-07T15:15:52,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:52,553 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:52,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:52,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:52,705 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-07T15:15:52,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:52,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. as already flushing 2024-12-07T15:15:52,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:52,705 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:52,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:52,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:52,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-07T15:15:52,824 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:52,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43912 deadline: 1733584612813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:52,824 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:52,824 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:52,824 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:52,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:52,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1733584612843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:52,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:52,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43928 deadline: 1733584612843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:52,857 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:52,857 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:52,857 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:52,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:52,857 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:52,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43956 deadline: 1733584612843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:52,857 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:52,857 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:52,857 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:52,857 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:52,857 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:52,858 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-07T15:15:52,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:52,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. as already flushing 2024-12-07T15:15:52,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:52,858 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:52,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:52,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:52,868 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/C/95e2a3008942455b9f9c04ca585ddf4d 2024-12-07T15:15:52,872 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/A/c2c30357eec34a4db091e67c5afdf270 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/c2c30357eec34a4db091e67c5afdf270 2024-12-07T15:15:52,875 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/c2c30357eec34a4db091e67c5afdf270, entries=200, sequenceid=246, filesize=14.2 K 2024-12-07T15:15:52,876 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/B/5f676a18ea6b4aedb984bd8d1ec08e1d as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/5f676a18ea6b4aedb984bd8d1ec08e1d 2024-12-07T15:15:52,878 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/5f676a18ea6b4aedb984bd8d1ec08e1d, entries=150, sequenceid=246, filesize=11.9 K 2024-12-07T15:15:52,879 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/C/95e2a3008942455b9f9c04ca585ddf4d as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/95e2a3008942455b9f9c04ca585ddf4d 2024-12-07T15:15:52,882 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/95e2a3008942455b9f9c04ca585ddf4d, entries=150, sequenceid=246, filesize=11.9 K 2024-12-07T15:15:52,882 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 54b89cbce0d0fe96d7c51211398eae76 in 846ms, sequenceid=246, compaction requested=true 2024-12-07T15:15:52,882 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 54b89cbce0d0fe96d7c51211398eae76: 2024-12-07T15:15:52,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54b89cbce0d0fe96d7c51211398eae76:A, priority=-2147483648, current under compaction store size is 1 2024-12-07T15:15:52,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:15:52,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54b89cbce0d0fe96d7c51211398eae76:B, priority=-2147483648, current under compaction store size is 2 2024-12-07T15:15:52,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:15:52,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54b89cbce0d0fe96d7c51211398eae76:C, priority=-2147483648, current under compaction store size is 3 2024-12-07T15:15:52,883 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:15:52,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T15:15:52,883 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:15:52,883 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36999 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:15:52,883 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41779 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:15:52,883 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1541): 54b89cbce0d0fe96d7c51211398eae76/B is initiating minor compaction (all files) 2024-12-07T15:15:52,883 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1541): 54b89cbce0d0fe96d7c51211398eae76/A is initiating minor compaction (all files) 2024-12-07T15:15:52,883 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54b89cbce0d0fe96d7c51211398eae76/B in TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:52,883 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54b89cbce0d0fe96d7c51211398eae76/A in TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:52,883 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/e263ea4699c6403682fc9f5f798b6e27, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/7f9b7eba129a47acbb98f9b7401c3539, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/5f676a18ea6b4aedb984bd8d1ec08e1d] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp, totalSize=36.1 K 2024-12-07T15:15:52,883 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/ec560318fd3c4a16bd05cef580798755, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/ad3cd1aaf0f04fed8470bbb5cd364179, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/c2c30357eec34a4db091e67c5afdf270] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp, totalSize=40.8 K 2024-12-07T15:15:52,884 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting e263ea4699c6403682fc9f5f798b6e27, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1733584549802 2024-12-07T15:15:52,884 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec560318fd3c4a16bd05cef580798755, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1733584549802 2024-12-07T15:15:52,884 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad3cd1aaf0f04fed8470bbb5cd364179, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1733584549920 2024-12-07T15:15:52,884 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f9b7eba129a47acbb98f9b7401c3539, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1733584549920 2024-12-07T15:15:52,884 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f676a18ea6b4aedb984bd8d1ec08e1d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1733584551174 2024-12-07T15:15:52,884 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting c2c30357eec34a4db091e67c5afdf270, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1733584551174 2024-12-07T15:15:52,890 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54b89cbce0d0fe96d7c51211398eae76#A#compaction#424 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:15:52,891 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/A/14c2c642404545f7bd1bda37dd0a7b39 is 50, key is test_row_0/A:col10/1733584552034/Put/seqid=0 2024-12-07T15:15:52,893 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54b89cbce0d0fe96d7c51211398eae76#B#compaction#425 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:15:52,893 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/B/6e79476335244b50be7af4f5aaa4a03e is 50, key is test_row_0/B:col10/1733584552034/Put/seqid=0 2024-12-07T15:15:52,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742322_1498 (size=12799) 2024-12-07T15:15:52,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742323_1499 (size=12799) 2024-12-07T15:15:53,011 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-07T15:15:53,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:53,011 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing 54b89cbce0d0fe96d7c51211398eae76 3/3 column families, dataSize=134.18 KB heapSize=366.23 KB 2024-12-07T15:15:53,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b89cbce0d0fe96d7c51211398eae76, store=A 2024-12-07T15:15:53,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T15:15:53,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b89cbce0d0fe96d7c51211398eae76, store=B 2024-12-07T15:15:53,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T15:15:53,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b89cbce0d0fe96d7c51211398eae76, store=C 2024-12-07T15:15:53,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T15:15:53,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/A/e0eca1148ef443568bc369ba7a5faf7b is 50, key is test_row_0/A:col10/1733584552176/Put/seqid=0 2024-12-07T15:15:53,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742324_1500 (size=12301) 2024-12-07T15:15:53,300 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/A/14c2c642404545f7bd1bda37dd0a7b39 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/14c2c642404545f7bd1bda37dd0a7b39 2024-12-07T15:15:53,304 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54b89cbce0d0fe96d7c51211398eae76/A of 54b89cbce0d0fe96d7c51211398eae76 into 14c2c642404545f7bd1bda37dd0a7b39(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:15:53,304 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54b89cbce0d0fe96d7c51211398eae76: 2024-12-07T15:15:53,304 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., storeName=54b89cbce0d0fe96d7c51211398eae76/A, priority=13, startTime=1733584552882; duration=0sec 2024-12-07T15:15:53,304 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T15:15:53,304 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54b89cbce0d0fe96d7c51211398eae76:A 2024-12-07T15:15:53,304 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:15:53,305 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36999 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:15:53,305 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1541): 54b89cbce0d0fe96d7c51211398eae76/C is initiating minor compaction (all files) 2024-12-07T15:15:53,305 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54b89cbce0d0fe96d7c51211398eae76/C in TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:53,305 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/e4afea5931d646f79c50a1ca325ce115, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/34831ea61f9f4084808d416b314b6ad1, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/95e2a3008942455b9f9c04ca585ddf4d] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp, totalSize=36.1 K 2024-12-07T15:15:53,306 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/B/6e79476335244b50be7af4f5aaa4a03e as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/6e79476335244b50be7af4f5aaa4a03e 2024-12-07T15:15:53,306 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting e4afea5931d646f79c50a1ca325ce115, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1733584549802 2024-12-07T15:15:53,307 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 34831ea61f9f4084808d416b314b6ad1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1733584549920 2024-12-07T15:15:53,307 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 95e2a3008942455b9f9c04ca585ddf4d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1733584551174 2024-12-07T15:15:53,311 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54b89cbce0d0fe96d7c51211398eae76/B of 54b89cbce0d0fe96d7c51211398eae76 into 6e79476335244b50be7af4f5aaa4a03e(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:15:53,311 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54b89cbce0d0fe96d7c51211398eae76: 2024-12-07T15:15:53,311 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., storeName=54b89cbce0d0fe96d7c51211398eae76/B, priority=13, startTime=1733584552883; duration=0sec 2024-12-07T15:15:53,311 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:15:53,311 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54b89cbce0d0fe96d7c51211398eae76:B 2024-12-07T15:15:53,314 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54b89cbce0d0fe96d7c51211398eae76#C#compaction#427 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:15:53,315 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/C/6d2d9b0d599e422f8d01389b26f002fe is 50, key is test_row_0/C:col10/1733584552034/Put/seqid=0 2024-12-07T15:15:53,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742325_1501 (size=12799) 2024-12-07T15:15:53,324 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/C/6d2d9b0d599e422f8d01389b26f002fe as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/6d2d9b0d599e422f8d01389b26f002fe 2024-12-07T15:15:53,327 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54b89cbce0d0fe96d7c51211398eae76/C of 54b89cbce0d0fe96d7c51211398eae76 into 6d2d9b0d599e422f8d01389b26f002fe(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:15:53,327 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54b89cbce0d0fe96d7c51211398eae76: 2024-12-07T15:15:53,327 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., storeName=54b89cbce0d0fe96d7c51211398eae76/C, priority=13, startTime=1733584552883; duration=0sec 2024-12-07T15:15:53,327 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:15:53,327 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54b89cbce0d0fe96d7c51211398eae76:C 2024-12-07T15:15:53,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on 54b89cbce0d0fe96d7c51211398eae76 2024-12-07T15:15:53,344 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. as already flushing 2024-12-07T15:15:53,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:53,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43928 deadline: 1733584613378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:53,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:53,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43956 deadline: 1733584613378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:53,395 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:53,395 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:53,396 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:53,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:53,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43952 deadline: 1733584613393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:53,396 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:53,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:53,396 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:53,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43912 deadline: 1733584613395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:53,396 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:53,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:53,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:53,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:53,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1733584613395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:53,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:53,396 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to d6797ebe9c52,46009,1733584418401 for 'test_row_2' in 54b89cbce0d0fe96d7c51211398eae76 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-07T15:15:53,396 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:53,397 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:53,397 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:53,397 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:53,397 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:53,397 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:53,421 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/A/e0eca1148ef443568bc369ba7a5faf7b 2024-12-07T15:15:53,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/B/1034eff85b2641baaf0e7629baeb7e98 is 50, key is test_row_0/B:col10/1733584552176/Put/seqid=0 2024-12-07T15:15:53,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742326_1502 (size=12301) 2024-12-07T15:15:53,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:53,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43928 deadline: 1733584613503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:53,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:53,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43956 deadline: 1733584613503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:53,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:53,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:53,516 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:53,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:53,516 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:53,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43912 deadline: 1733584613503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:53,516 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:53,516 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:53,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:53,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:53,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1733584613504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:53,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:53,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:53,516 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:53,516 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:53,516 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:53,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:53,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43928 deadline: 1733584613723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:53,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:53,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43956 deadline: 1733584613723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:53,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:53,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:53,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:53,733 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:53,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:53,733 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:53,733 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:53,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43912 deadline: 1733584613723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:53,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:53,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:53,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1733584613724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:53,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:53,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:53,734 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:53,734 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:53,734 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:53,831 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/B/1034eff85b2641baaf0e7629baeb7e98 2024-12-07T15:15:53,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/C/74981f233a8d41f5b1ab4ad26316b7da is 50, key is test_row_0/C:col10/1733584552176/Put/seqid=0 2024-12-07T15:15:53,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742327_1503 (size=12301) 2024-12-07T15:15:54,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:54,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43928 deadline: 1733584614043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:54,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:54,056 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:54,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43956 deadline: 1733584614043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:54,056 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:54,056 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:54,056 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:54,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:54,056 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:54,056 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:54,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43912 deadline: 1733584614043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:54,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:54,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:54,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:54,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:54,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1733584614043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:54,057 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:54,057 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:54,057 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:54,243 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/C/74981f233a8d41f5b1ab4ad26316b7da 2024-12-07T15:15:54,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/A/e0eca1148ef443568bc369ba7a5faf7b as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/e0eca1148ef443568bc369ba7a5faf7b 2024-12-07T15:15:54,251 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/e0eca1148ef443568bc369ba7a5faf7b, entries=150, sequenceid=279, filesize=12.0 K 2024-12-07T15:15:54,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/B/1034eff85b2641baaf0e7629baeb7e98 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/1034eff85b2641baaf0e7629baeb7e98 2024-12-07T15:15:54,254 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/1034eff85b2641baaf0e7629baeb7e98, entries=150, sequenceid=279, filesize=12.0 K 2024-12-07T15:15:54,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/C/74981f233a8d41f5b1ab4ad26316b7da as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/74981f233a8d41f5b1ab4ad26316b7da 2024-12-07T15:15:54,257 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/74981f233a8d41f5b1ab4ad26316b7da, entries=150, sequenceid=279, filesize=12.0 K 2024-12-07T15:15:54,258 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~366.19 KB/374976, currentSize=60.38 KB/61830 for 54b89cbce0d0fe96d7c51211398eae76 in 1247ms, sequenceid=279, compaction requested=false 2024-12-07T15:15:54,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for 54b89cbce0d0fe96d7c51211398eae76: 2024-12-07T15:15:54,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:54,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-12-07T15:15:54,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-12-07T15:15:54,261 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-12-07T15:15:54,261 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.6940 sec 2024-12-07T15:15:54,263 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 3.6980 sec 2024-12-07T15:15:54,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on 54b89cbce0d0fe96d7c51211398eae76 2024-12-07T15:15:54,567 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 54b89cbce0d0fe96d7c51211398eae76 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-07T15:15:54,568 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b89cbce0d0fe96d7c51211398eae76, store=A 2024-12-07T15:15:54,568 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:15:54,568 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b89cbce0d0fe96d7c51211398eae76, store=B 2024-12-07T15:15:54,568 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:15:54,568 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b89cbce0d0fe96d7c51211398eae76, store=C 2024-12-07T15:15:54,568 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:15:54,572 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/A/88493b2dcb7543708a62fb48225d5960 is 50, key is test_row_0/A:col10/1733584553377/Put/seqid=0 2024-12-07T15:15:54,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742328_1504 (size=14741) 2024-12-07T15:15:54,577 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/A/88493b2dcb7543708a62fb48225d5960 2024-12-07T15:15:54,588 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/B/c236134de5cd47e289708b41b314a2b5 is 50, key is test_row_0/B:col10/1733584553377/Put/seqid=0 2024-12-07T15:15:54,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742329_1505 (size=12301) 2024-12-07T15:15:54,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:54,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43956 deadline: 1733584614606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:54,616 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:54,616 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:54,616 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:54,620 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:54,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1733584614608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:54,620 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:54,620 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:54,620 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:54,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:54,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43912 deadline: 1733584614613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:54,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:54,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43928 deadline: 1733584614614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:54,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:54,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:54,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:54,627 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:54,627 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:54,627 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:54,729 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:54,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43956 deadline: 1733584614723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:54,729 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:54,729 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:54,729 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:54,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1733584614723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:54,729 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:54,730 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:54,730 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:54,730 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:54,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-07T15:15:54,733 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T15:15:54,734 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T15:15:54,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees 2024-12-07T15:15:54,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-07T15:15:54,735 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T15:15:54,736 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T15:15:54,736 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T15:15:54,742 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:54,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43912 deadline: 1733584614733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:54,742 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:54,742 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:54,742 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:54,742 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:54,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43928 deadline: 1733584614733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:54,743 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:54,743 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:54,743 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:54,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-07T15:15:54,888 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-12-07T15:15:54,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:54,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. as already flushing 2024-12-07T15:15:54,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:54,889 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:54,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:54,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:54,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:54,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43956 deadline: 1733584614933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:54,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:54,944 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:54,944 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:54,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1733584614933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:54,944 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:54,944 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:54,944 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:54,944 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:54,957 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:54,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43912 deadline: 1733584614953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:54,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:54,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:54,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43928 deadline: 1733584614954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:54,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:54,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:54,958 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:54,958 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:54,958 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:54,991 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/B/c236134de5cd47e289708b41b314a2b5 2024-12-07T15:15:54,999 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/C/2a6a9018366149bbb051746a4cdd1b99 is 50, key is test_row_0/C:col10/1733584553377/Put/seqid=0 2024-12-07T15:15:55,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742330_1506 (size=12301) 2024-12-07T15:15:55,002 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/C/2a6a9018366149bbb051746a4cdd1b99 2024-12-07T15:15:55,005 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/A/88493b2dcb7543708a62fb48225d5960 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/88493b2dcb7543708a62fb48225d5960 2024-12-07T15:15:55,008 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/88493b2dcb7543708a62fb48225d5960, entries=200, sequenceid=295, filesize=14.4 K 2024-12-07T15:15:55,009 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/B/c236134de5cd47e289708b41b314a2b5 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/c236134de5cd47e289708b41b314a2b5 2024-12-07T15:15:55,012 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/c236134de5cd47e289708b41b314a2b5, entries=150, sequenceid=295, filesize=12.0 K 2024-12-07T15:15:55,013 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/C/2a6a9018366149bbb051746a4cdd1b99 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/2a6a9018366149bbb051746a4cdd1b99 2024-12-07T15:15:55,015 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/2a6a9018366149bbb051746a4cdd1b99, entries=150, sequenceid=295, filesize=12.0 K 2024-12-07T15:15:55,016 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 54b89cbce0d0fe96d7c51211398eae76 in 449ms, sequenceid=295, compaction requested=true 2024-12-07T15:15:55,016 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 54b89cbce0d0fe96d7c51211398eae76: 2024-12-07T15:15:55,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54b89cbce0d0fe96d7c51211398eae76:A, priority=-2147483648, current under compaction store size is 1 2024-12-07T15:15:55,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:15:55,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54b89cbce0d0fe96d7c51211398eae76:B, priority=-2147483648, current under compaction store size is 2 2024-12-07T15:15:55,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:15:55,016 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:15:55,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54b89cbce0d0fe96d7c51211398eae76:C, priority=-2147483648, current under compaction store size is 3 2024-12-07T15:15:55,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T15:15:55,017 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:15:55,017 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39841 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:15:55,017 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37401 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:15:55,017 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1541): 54b89cbce0d0fe96d7c51211398eae76/A is initiating minor compaction (all files) 2024-12-07T15:15:55,017 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1541): 54b89cbce0d0fe96d7c51211398eae76/B is initiating minor compaction (all files) 2024-12-07T15:15:55,017 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54b89cbce0d0fe96d7c51211398eae76/A in TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:55,017 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54b89cbce0d0fe96d7c51211398eae76/B in TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:55,017 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/14c2c642404545f7bd1bda37dd0a7b39, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/e0eca1148ef443568bc369ba7a5faf7b, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/88493b2dcb7543708a62fb48225d5960] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp, totalSize=38.9 K 2024-12-07T15:15:55,017 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/6e79476335244b50be7af4f5aaa4a03e, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/1034eff85b2641baaf0e7629baeb7e98, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/c236134de5cd47e289708b41b314a2b5] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp, totalSize=36.5 K 2024-12-07T15:15:55,018 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e79476335244b50be7af4f5aaa4a03e, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1733584551174 2024-12-07T15:15:55,018 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14c2c642404545f7bd1bda37dd0a7b39, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1733584551174 2024-12-07T15:15:55,018 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting e0eca1148ef443568bc369ba7a5faf7b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1733584552174 2024-12-07T15:15:55,018 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 1034eff85b2641baaf0e7629baeb7e98, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1733584552174 2024-12-07T15:15:55,018 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 88493b2dcb7543708a62fb48225d5960, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1733584553345 2024-12-07T15:15:55,018 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting c236134de5cd47e289708b41b314a2b5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1733584553345 2024-12-07T15:15:55,025 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54b89cbce0d0fe96d7c51211398eae76#B#compaction#433 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:15:55,025 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54b89cbce0d0fe96d7c51211398eae76#A#compaction#434 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:15:55,025 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/B/3441a348dfc04f99ad62cc736dd51245 is 50, key is test_row_0/B:col10/1733584553377/Put/seqid=0 2024-12-07T15:15:55,025 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/A/bd0e603fc2f0444391957559010e18a3 is 50, key is test_row_0/A:col10/1733584553377/Put/seqid=0 2024-12-07T15:15:55,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742331_1507 (size=13051) 2024-12-07T15:15:55,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742332_1508 (size=13051) 2024-12-07T15:15:55,032 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/B/3441a348dfc04f99ad62cc736dd51245 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/3441a348dfc04f99ad62cc736dd51245 2024-12-07T15:15:55,038 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54b89cbce0d0fe96d7c51211398eae76/B of 54b89cbce0d0fe96d7c51211398eae76 into 3441a348dfc04f99ad62cc736dd51245(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:15:55,038 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54b89cbce0d0fe96d7c51211398eae76: 2024-12-07T15:15:55,038 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., storeName=54b89cbce0d0fe96d7c51211398eae76/B, priority=13, startTime=1733584555016; duration=0sec 2024-12-07T15:15:55,038 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T15:15:55,038 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54b89cbce0d0fe96d7c51211398eae76:B 2024-12-07T15:15:55,038 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:15:55,039 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37401 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:15:55,039 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1541): 54b89cbce0d0fe96d7c51211398eae76/C is initiating minor compaction (all files) 2024-12-07T15:15:55,039 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54b89cbce0d0fe96d7c51211398eae76/C in TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:55,039 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/6d2d9b0d599e422f8d01389b26f002fe, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/74981f233a8d41f5b1ab4ad26316b7da, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/2a6a9018366149bbb051746a4cdd1b99] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp, totalSize=36.5 K 2024-12-07T15:15:55,039 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d2d9b0d599e422f8d01389b26f002fe, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1733584551174 2024-12-07T15:15:55,039 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 74981f233a8d41f5b1ab4ad26316b7da, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1733584552174 2024-12-07T15:15:55,040 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a6a9018366149bbb051746a4cdd1b99, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1733584553345 2024-12-07T15:15:55,041 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-12-07T15:15:55,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:55,041 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2902): Flushing 54b89cbce0d0fe96d7c51211398eae76 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-07T15:15:55,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b89cbce0d0fe96d7c51211398eae76, store=A 2024-12-07T15:15:55,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:15:55,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b89cbce0d0fe96d7c51211398eae76, store=B 2024-12-07T15:15:55,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:15:55,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b89cbce0d0fe96d7c51211398eae76, store=C 2024-12-07T15:15:55,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:15:55,047 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54b89cbce0d0fe96d7c51211398eae76#C#compaction#435 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:15:55,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/A/e8b0f3fad0224516a056945dd0f77b73 is 50, key is test_row_0/A:col10/1733584554613/Put/seqid=0 2024-12-07T15:15:55,048 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/C/049ea8d678b944eeb16963628d8e44f0 is 50, key is test_row_0/C:col10/1733584553377/Put/seqid=0 2024-12-07T15:15:55,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742333_1509 (size=13051) 2024-12-07T15:15:55,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742334_1510 (size=12301) 2024-12-07T15:15:55,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-07T15:15:55,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on 54b89cbce0d0fe96d7c51211398eae76 2024-12-07T15:15:55,259 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. as already flushing 2024-12-07T15:15:55,285 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:55,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1733584615273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:55,285 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:55,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43956 deadline: 1733584615280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:55,285 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:55,285 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:55,285 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:55,286 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:55,286 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:55,286 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:55,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:55,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43912 deadline: 1733584615283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:55,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:55,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:55,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:55,296 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:55,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43928 deadline: 1733584615284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:55,297 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:55,297 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:55,297 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:55,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-07T15:15:55,401 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:55,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1733584615393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:55,402 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:55,402 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:55,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43956 deadline: 1733584615393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:55,402 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:55,402 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:55,402 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:55,402 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:55,402 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:55,412 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:55,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43952 deadline: 1733584615403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:55,412 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:55,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43912 deadline: 1733584615403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:55,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:55,413 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:55,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:55,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:55,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43928 deadline: 1733584615403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:55,413 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to d6797ebe9c52,46009,1733584418401 for 'test_row_2' in 54b89cbce0d0fe96d7c51211398eae76 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4206 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-07T15:15:55,413 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:55,413 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:55,413 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:55,413 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:55,413 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:55,413 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:55,433 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/A/bd0e603fc2f0444391957559010e18a3 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/bd0e603fc2f0444391957559010e18a3 2024-12-07T15:15:55,437 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54b89cbce0d0fe96d7c51211398eae76/A of 54b89cbce0d0fe96d7c51211398eae76 into bd0e603fc2f0444391957559010e18a3(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:15:55,437 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54b89cbce0d0fe96d7c51211398eae76: 2024-12-07T15:15:55,437 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., storeName=54b89cbce0d0fe96d7c51211398eae76/A, priority=13, startTime=1733584555016; duration=0sec 2024-12-07T15:15:55,437 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:15:55,437 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54b89cbce0d0fe96d7c51211398eae76:A 2024-12-07T15:15:55,452 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/A/e8b0f3fad0224516a056945dd0f77b73 2024-12-07T15:15:55,455 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/C/049ea8d678b944eeb16963628d8e44f0 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/049ea8d678b944eeb16963628d8e44f0 2024-12-07T15:15:55,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/B/b1c1965820794d8e93f5de0d6a08a958 is 50, key is test_row_0/B:col10/1733584554613/Put/seqid=0 2024-12-07T15:15:55,459 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54b89cbce0d0fe96d7c51211398eae76/C of 54b89cbce0d0fe96d7c51211398eae76 into 049ea8d678b944eeb16963628d8e44f0(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:15:55,459 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54b89cbce0d0fe96d7c51211398eae76: 2024-12-07T15:15:55,459 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., storeName=54b89cbce0d0fe96d7c51211398eae76/C, priority=13, startTime=1733584555016; duration=0sec 2024-12-07T15:15:55,459 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:15:55,459 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54b89cbce0d0fe96d7c51211398eae76:C 2024-12-07T15:15:55,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742335_1511 (size=12301) 2024-12-07T15:15:55,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:55,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1733584615613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:55,623 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:55,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43956 deadline: 1733584615613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:55,623 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:55,623 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:55,623 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:55,623 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:55,623 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:55,623 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:55,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:55,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43912 deadline: 1733584615623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:55,630 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:55,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43928 deadline: 1733584615623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:55,630 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:55,630 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:55,630 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:55,630 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:55,630 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:55,630 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:55,862 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/B/b1c1965820794d8e93f5de0d6a08a958 2024-12-07T15:15:55,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/C/533df453e2ed454f8a5403fc5e30ab87 is 50, key is test_row_0/C:col10/1733584554613/Put/seqid=0 2024-12-07T15:15:55,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-07T15:15:55,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742336_1512 (size=12301) 2024-12-07T15:15:55,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:55,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1733584615933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:55,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:55,941 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:55,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43956 deadline: 1733584615933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:55,941 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:55,941 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:55,941 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:55,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:55,941 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:55,941 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:55,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43928 deadline: 1733584615933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:55,941 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:55,941 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:55,941 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:55,952 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:55,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43912 deadline: 1733584615943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:55,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:55,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:55,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:56,277 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/C/533df453e2ed454f8a5403fc5e30ab87 2024-12-07T15:15:56,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/A/e8b0f3fad0224516a056945dd0f77b73 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/e8b0f3fad0224516a056945dd0f77b73 2024-12-07T15:15:56,285 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/e8b0f3fad0224516a056945dd0f77b73, entries=150, sequenceid=315, filesize=12.0 K 2024-12-07T15:15:56,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/B/b1c1965820794d8e93f5de0d6a08a958 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/b1c1965820794d8e93f5de0d6a08a958 2024-12-07T15:15:56,290 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/b1c1965820794d8e93f5de0d6a08a958, entries=150, sequenceid=315, filesize=12.0 K 2024-12-07T15:15:56,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/C/533df453e2ed454f8a5403fc5e30ab87 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/533df453e2ed454f8a5403fc5e30ab87 2024-12-07T15:15:56,294 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/533df453e2ed454f8a5403fc5e30ab87, entries=150, sequenceid=315, filesize=12.0 K 2024-12-07T15:15:56,294 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 54b89cbce0d0fe96d7c51211398eae76 in 1253ms, sequenceid=315, compaction requested=false 2024-12-07T15:15:56,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2603): Flush status journal for 54b89cbce0d0fe96d7c51211398eae76: 2024-12-07T15:15:56,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:56,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=138 2024-12-07T15:15:56,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster(4169): Remote procedure done, pid=138 2024-12-07T15:15:56,297 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-12-07T15:15:56,297 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5600 sec 2024-12-07T15:15:56,299 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees in 1.5640 sec 2024-12-07T15:15:56,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on 54b89cbce0d0fe96d7c51211398eae76 2024-12-07T15:15:56,457 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 54b89cbce0d0fe96d7c51211398eae76 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-07T15:15:56,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b89cbce0d0fe96d7c51211398eae76, store=A 2024-12-07T15:15:56,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:15:56,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b89cbce0d0fe96d7c51211398eae76, store=B 2024-12-07T15:15:56,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:15:56,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b89cbce0d0fe96d7c51211398eae76, store=C 2024-12-07T15:15:56,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:15:56,462 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/A/37af6e60eebf460e925e7ad2fbe0fbf6 is 50, key is test_row_0/A:col10/1733584556456/Put/seqid=0 2024-12-07T15:15:56,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742337_1513 (size=14741) 2024-12-07T15:15:56,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:56,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43956 deadline: 1733584616490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:56,501 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:56,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:56,502 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:56,502 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:56,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43928 deadline: 1733584616491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:56,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:56,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:56,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:56,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:56,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1733584616492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:56,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:56,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:56,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:56,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:56,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43912 deadline: 1733584616501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:56,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:56,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:56,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:56,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:56,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43956 deadline: 1733584616613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:56,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:56,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43928 deadline: 1733584616613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:56,622 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:56,622 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:56,622 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:56,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:56,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1733584616613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:56,622 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:56,622 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:56,622 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:56,623 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:56,623 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:56,623 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:56,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:56,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43912 deadline: 1733584616623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:56,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:56,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:56,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:56,845 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:56,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43956 deadline: 1733584616833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:56,846 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:56,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:56,846 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:56,846 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:56,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43928 deadline: 1733584616833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:56,846 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:56,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:56,846 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:56,846 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:56,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1733584616833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:56,847 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:56,847 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:56,847 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:56,852 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:56,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43912 deadline: 1733584616843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:56,853 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:56,853 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:56,853 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:56,870 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/A/37af6e60eebf460e925e7ad2fbe0fbf6 2024-12-07T15:15:56,877 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/B/526c01506d2141f49e8ced2ab4205320 is 50, key is test_row_0/B:col10/1733584556456/Put/seqid=0 2024-12-07T15:15:56,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742338_1514 (size=12301) 2024-12-07T15:15:56,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-07T15:15:56,883 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T15:15:56,884 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T15:15:56,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees 2024-12-07T15:15:56,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-12-07T15:15:56,885 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T15:15:56,886 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T15:15:56,886 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T15:15:56,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-12-07T15:15:57,038 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-12-07T15:15:57,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:57,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. as already flushing 2024-12-07T15:15:57,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:57,038 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:57,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:57,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:57,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:57,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43956 deadline: 1733584617153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:57,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:57,161 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:57,161 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:57,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43928 deadline: 1733584617154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:57,161 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:57,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:57,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1733584617154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:57,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:57,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:57,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:57,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:57,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:57,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:57,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:57,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43912 deadline: 1733584617163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:57,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:57,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:57,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:57,191 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-12-07T15:15:57,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:57,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. as already flushing 2024-12-07T15:15:57,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:57,192 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:57,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:57,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:57,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-12-07T15:15:57,284 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/B/526c01506d2141f49e8ced2ab4205320 2024-12-07T15:15:57,294 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/C/7af8bb18aaca42aabea4f6eaa66a2805 is 50, key is test_row_0/C:col10/1733584556456/Put/seqid=0 2024-12-07T15:15:57,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742339_1515 (size=12301) 2024-12-07T15:15:57,314 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/C/7af8bb18aaca42aabea4f6eaa66a2805 2024-12-07T15:15:57,319 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/A/37af6e60eebf460e925e7ad2fbe0fbf6 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/37af6e60eebf460e925e7ad2fbe0fbf6 2024-12-07T15:15:57,322 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/37af6e60eebf460e925e7ad2fbe0fbf6, entries=200, sequenceid=331, filesize=14.4 K 2024-12-07T15:15:57,323 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/B/526c01506d2141f49e8ced2ab4205320 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/526c01506d2141f49e8ced2ab4205320 2024-12-07T15:15:57,326 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/526c01506d2141f49e8ced2ab4205320, entries=150, sequenceid=331, filesize=12.0 K 2024-12-07T15:15:57,327 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/C/7af8bb18aaca42aabea4f6eaa66a2805 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/7af8bb18aaca42aabea4f6eaa66a2805 2024-12-07T15:15:57,334 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/7af8bb18aaca42aabea4f6eaa66a2805, entries=150, sequenceid=331, filesize=12.0 K 2024-12-07T15:15:57,335 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 54b89cbce0d0fe96d7c51211398eae76 in 878ms, sequenceid=331, compaction requested=true 2024-12-07T15:15:57,335 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 54b89cbce0d0fe96d7c51211398eae76: 2024-12-07T15:15:57,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54b89cbce0d0fe96d7c51211398eae76:A, priority=-2147483648, current under compaction store size is 1 2024-12-07T15:15:57,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:15:57,335 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:15:57,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54b89cbce0d0fe96d7c51211398eae76:B, priority=-2147483648, current under compaction store size is 2 2024-12-07T15:15:57,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:15:57,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54b89cbce0d0fe96d7c51211398eae76:C, priority=-2147483648, current under compaction store size is 3 2024-12-07T15:15:57,336 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:15:57,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T15:15:57,337 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:15:57,337 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1541): 54b89cbce0d0fe96d7c51211398eae76/B is initiating minor compaction (all files) 2024-12-07T15:15:57,337 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54b89cbce0d0fe96d7c51211398eae76/B in TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:57,337 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/3441a348dfc04f99ad62cc736dd51245, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/b1c1965820794d8e93f5de0d6a08a958, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/526c01506d2141f49e8ced2ab4205320] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp, totalSize=36.8 K 2024-12-07T15:15:57,337 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40093 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:15:57,337 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1541): 54b89cbce0d0fe96d7c51211398eae76/A is initiating minor compaction (all files) 2024-12-07T15:15:57,337 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54b89cbce0d0fe96d7c51211398eae76/A in TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:57,337 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/bd0e603fc2f0444391957559010e18a3, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/e8b0f3fad0224516a056945dd0f77b73, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/37af6e60eebf460e925e7ad2fbe0fbf6] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp, totalSize=39.2 K 2024-12-07T15:15:57,337 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd0e603fc2f0444391957559010e18a3, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1733584553345 2024-12-07T15:15:57,338 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 3441a348dfc04f99ad62cc736dd51245, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1733584553345 2024-12-07T15:15:57,338 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting e8b0f3fad0224516a056945dd0f77b73, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1733584554594 2024-12-07T15:15:57,338 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting b1c1965820794d8e93f5de0d6a08a958, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1733584554594 2024-12-07T15:15:57,338 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 37af6e60eebf460e925e7ad2fbe0fbf6, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1733584555278 2024-12-07T15:15:57,338 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 526c01506d2141f49e8ced2ab4205320, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1733584555278 2024-12-07T15:15:57,345 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-12-07T15:15:57,346 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54b89cbce0d0fe96d7c51211398eae76#A#compaction#442 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:15:57,347 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/A/92c28d5fd2c446c7aa0303def717e493 is 50, key is test_row_0/A:col10/1733584556456/Put/seqid=0 2024-12-07T15:15:57,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:57,347 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2902): Flushing 54b89cbce0d0fe96d7c51211398eae76 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-07T15:15:57,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b89cbce0d0fe96d7c51211398eae76, store=A 2024-12-07T15:15:57,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:15:57,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b89cbce0d0fe96d7c51211398eae76, store=B 2024-12-07T15:15:57,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:15:57,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b89cbce0d0fe96d7c51211398eae76, store=C 2024-12-07T15:15:57,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:15:57,357 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54b89cbce0d0fe96d7c51211398eae76#B#compaction#443 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:15:57,357 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/B/b2f45ce89b094fee899952aa129c8f34 is 50, key is test_row_0/B:col10/1733584556456/Put/seqid=0 2024-12-07T15:15:57,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/A/3d71f26c7fb34ed58b620b430d6910a6 is 50, key is test_row_0/A:col10/1733584556488/Put/seqid=0 2024-12-07T15:15:57,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742340_1516 (size=13153) 2024-12-07T15:15:57,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742341_1517 (size=13153) 2024-12-07T15:15:57,384 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/B/b2f45ce89b094fee899952aa129c8f34 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/b2f45ce89b094fee899952aa129c8f34 2024-12-07T15:15:57,389 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54b89cbce0d0fe96d7c51211398eae76/B of 54b89cbce0d0fe96d7c51211398eae76 into b2f45ce89b094fee899952aa129c8f34(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:15:57,389 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54b89cbce0d0fe96d7c51211398eae76: 2024-12-07T15:15:57,389 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., storeName=54b89cbce0d0fe96d7c51211398eae76/B, priority=13, startTime=1733584557335; duration=0sec 2024-12-07T15:15:57,389 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T15:15:57,389 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54b89cbce0d0fe96d7c51211398eae76:B 2024-12-07T15:15:57,389 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:15:57,390 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:15:57,390 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1541): 54b89cbce0d0fe96d7c51211398eae76/C is initiating minor compaction (all files) 2024-12-07T15:15:57,390 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54b89cbce0d0fe96d7c51211398eae76/C in TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:57,390 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/049ea8d678b944eeb16963628d8e44f0, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/533df453e2ed454f8a5403fc5e30ab87, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/7af8bb18aaca42aabea4f6eaa66a2805] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp, totalSize=36.8 K 2024-12-07T15:15:57,390 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 049ea8d678b944eeb16963628d8e44f0, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1733584553345 2024-12-07T15:15:57,391 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 533df453e2ed454f8a5403fc5e30ab87, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1733584554594 2024-12-07T15:15:57,393 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 7af8bb18aaca42aabea4f6eaa66a2805, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1733584555278 2024-12-07T15:15:57,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742342_1518 (size=12301) 2024-12-07T15:15:57,401 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/A/3d71f26c7fb34ed58b620b430d6910a6 2024-12-07T15:15:57,402 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54b89cbce0d0fe96d7c51211398eae76#C#compaction#445 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:15:57,403 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/C/5db057dda44c4fb3ba05d6c5292e2036 is 50, key is test_row_0/C:col10/1733584556456/Put/seqid=0 2024-12-07T15:15:57,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/B/2e27d42aea2d4124adf26c2a4e5e1d51 is 50, key is test_row_0/B:col10/1733584556488/Put/seqid=0 2024-12-07T15:15:57,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742343_1519 (size=13153) 2024-12-07T15:15:57,429 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/C/5db057dda44c4fb3ba05d6c5292e2036 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/5db057dda44c4fb3ba05d6c5292e2036 2024-12-07T15:15:57,437 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54b89cbce0d0fe96d7c51211398eae76/C of 54b89cbce0d0fe96d7c51211398eae76 into 5db057dda44c4fb3ba05d6c5292e2036(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:15:57,437 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54b89cbce0d0fe96d7c51211398eae76: 2024-12-07T15:15:57,437 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., storeName=54b89cbce0d0fe96d7c51211398eae76/C, priority=13, startTime=1733584557336; duration=0sec 2024-12-07T15:15:57,437 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:15:57,437 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54b89cbce0d0fe96d7c51211398eae76:C 2024-12-07T15:15:57,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742344_1520 (size=12301) 2024-12-07T15:15:57,444 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/B/2e27d42aea2d4124adf26c2a4e5e1d51 2024-12-07T15:15:57,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/C/4a4052a53ba14869b783409776075bb8 is 50, key is test_row_0/C:col10/1733584556488/Put/seqid=0 2024-12-07T15:15:57,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742345_1521 (size=12301) 2024-12-07T15:15:57,472 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/C/4a4052a53ba14869b783409776075bb8 2024-12-07T15:15:57,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/A/3d71f26c7fb34ed58b620b430d6910a6 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/3d71f26c7fb34ed58b620b430d6910a6 2024-12-07T15:15:57,482 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/3d71f26c7fb34ed58b620b430d6910a6, entries=150, sequenceid=349, filesize=12.0 K 2024-12-07T15:15:57,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/B/2e27d42aea2d4124adf26c2a4e5e1d51 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/2e27d42aea2d4124adf26c2a4e5e1d51 2024-12-07T15:15:57,490 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/2e27d42aea2d4124adf26c2a4e5e1d51, entries=150, sequenceid=349, filesize=12.0 K 2024-12-07T15:15:57,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/C/4a4052a53ba14869b783409776075bb8 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/4a4052a53ba14869b783409776075bb8 2024-12-07T15:15:57,497 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/4a4052a53ba14869b783409776075bb8, entries=150, sequenceid=349, filesize=12.0 K 2024-12-07T15:15:57,498 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=0 B/0 for 54b89cbce0d0fe96d7c51211398eae76 in 150ms, sequenceid=349, compaction requested=false 2024-12-07T15:15:57,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2603): Flush status journal for 54b89cbce0d0fe96d7c51211398eae76: 2024-12-07T15:15:57,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:57,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=140 2024-12-07T15:15:57,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster(4169): Remote procedure done, pid=140 2024-12-07T15:15:57,504 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-12-07T15:15:57,504 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 614 msec 2024-12-07T15:15:57,507 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees in 621 msec 2024-12-07T15:15:57,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-12-07T15:15:57,513 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T15:15:57,514 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T15:15:57,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees 2024-12-07T15:15:57,516 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=141, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T15:15:57,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-12-07T15:15:57,517 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=141, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T15:15:57,517 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T15:15:57,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-12-07T15:15:57,670 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-12-07T15:15:57,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:57,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2603): Flush status journal for 54b89cbce0d0fe96d7c51211398eae76: 2024-12-07T15:15:57,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:57,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=142 2024-12-07T15:15:57,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster(4169): Remote procedure done, pid=142 2024-12-07T15:15:57,674 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-12-07T15:15:57,674 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 154 msec 2024-12-07T15:15:57,675 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees in 160 msec 2024-12-07T15:15:57,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on 54b89cbce0d0fe96d7c51211398eae76 2024-12-07T15:15:57,683 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 54b89cbce0d0fe96d7c51211398eae76 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-07T15:15:57,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b89cbce0d0fe96d7c51211398eae76, store=A 2024-12-07T15:15:57,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:15:57,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b89cbce0d0fe96d7c51211398eae76, store=B 2024-12-07T15:15:57,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:15:57,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b89cbce0d0fe96d7c51211398eae76, store=C 2024-12-07T15:15:57,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:15:57,687 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/A/ff0bdcb4fa9b4d2d8803f798c79a8f1d is 50, key is test_row_0/A:col10/1733584557678/Put/seqid=0 2024-12-07T15:15:57,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742346_1522 (size=14741) 2024-12-07T15:15:57,733 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733584557710, max timestamp=1733584557729Num uniques 859; 2024-12-07T15:15:57,733 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-07T15:15:57,733 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733584557710, max timestamp=1733584557729Num uniques 859; 2024-12-07T15:15:57,733 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-07T15:15:57,733 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733584557710, max timestamp=1733584557729Num uniques 859; 2024-12-07T15:15:57,733 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-07T15:15:57,779 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/A/92c28d5fd2c446c7aa0303def717e493 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/92c28d5fd2c446c7aa0303def717e493 2024-12-07T15:15:57,782 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54b89cbce0d0fe96d7c51211398eae76/A of 54b89cbce0d0fe96d7c51211398eae76 into 92c28d5fd2c446c7aa0303def717e493(size=12.8 K), total size for store is 24.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:15:57,782 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54b89cbce0d0fe96d7c51211398eae76: 2024-12-07T15:15:57,782 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., storeName=54b89cbce0d0fe96d7c51211398eae76/A, priority=13, startTime=1733584557335; duration=0sec 2024-12-07T15:15:57,782 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:15:57,782 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54b89cbce0d0fe96d7c51211398eae76:A 2024-12-07T15:15:57,793 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:57,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43928 deadline: 1733584617775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:57,793 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:57,793 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:57,793 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:57,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:57,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1733584617776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:57,795 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:57,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:57,795 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:57,795 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:57,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43956 deadline: 1733584617788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:57,796 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:57,796 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:57,796 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:57,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:57,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43912 deadline: 1733584617789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:57,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:57,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:57,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:57,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-12-07T15:15:57,833 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T15:15:57,834 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T15:15:57,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] procedure2.ProcedureExecutor(1139): Stored pid=143, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees 2024-12-07T15:15:57,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-12-07T15:15:57,835 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=143, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T15:15:57,836 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=143, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T15:15:57,836 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T15:15:57,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:57,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43928 deadline: 1733584617903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:57,917 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:57,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1733584617903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:57,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:57,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:57,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:57,917 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:57,917 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:57,917 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:57,917 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:57,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43956 deadline: 1733584617903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:57,917 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:57,917 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:57,918 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:57,924 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:57,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43912 deadline: 1733584617913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:57,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:57,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:57,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:57,936 INFO [Thread-1986 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1986. 2024-12-07T15:15:57,937 DEBUG [Thread-1986 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T15:15:57,937 DEBUG [Thread-1986 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:15:57,937 DEBUG [Thread-1986 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:15:57,937 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T15:15:57,942 INFO [Thread-1992 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1992. 2024-12-07T15:15:57,942 DEBUG [Thread-1992 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T15:15:57,942 DEBUG [Thread-1992 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:15:57,942 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T15:15:57,943 DEBUG [Thread-1992 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:15:57,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-12-07T15:15:57,945 INFO [Thread-1990 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1990. 2024-12-07T15:15:57,945 DEBUG [Thread-1990 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T15:15:57,945 DEBUG [Thread-1990 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:15:57,945 DEBUG [Thread-1990 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:15:57,945 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T15:15:57,951 INFO [Thread-1988 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1988. 2024-12-07T15:15:57,951 DEBUG [Thread-1988 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T15:15:57,951 DEBUG [Thread-1988 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:15:57,951 DEBUG [Thread-1988 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:15:57,951 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T15:15:57,954 INFO [Thread-1994 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1994. 2024-12-07T15:15:57,954 DEBUG [Thread-1994 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T15:15:57,954 DEBUG [Thread-1994 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:15:57,955 DEBUG [Thread-1994 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:15:57,955 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T15:15:57,988 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-12-07T15:15:57,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:57,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. as already flushing 2024-12-07T15:15:57,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:57,988 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] handler.RSProcedureHandler(58): pid=144 java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:57,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=144 java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:57,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=144 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:58,091 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/A/ff0bdcb4fa9b4d2d8803f798c79a8f1d 2024-12-07T15:15:58,096 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/B/ea2efeee56f24ad2a4aaaed2ac9b404b is 50, key is test_row_0/B:col10/1733584557678/Put/seqid=0 2024-12-07T15:15:58,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742347_1523 (size=12301) 2024-12-07T15:15:58,123 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:58,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:58,123 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:58,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43928 deadline: 1733584618123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:58,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1733584618123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:58,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43956 deadline: 1733584618123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:58,124 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:58,124 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:58,124 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:58,124 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:58,124 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:58,124 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:58,124 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:58,124 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:58,124 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:58,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:58,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43912 deadline: 1733584618133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:58,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:58,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:58,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:58,141 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-12-07T15:15:58,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:58,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. as already flushing 2024-12-07T15:15:58,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:58,141 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] handler.RSProcedureHandler(58): pid=144 java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:58,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=144 java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:58,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=144 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:58,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-12-07T15:15:58,293 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-12-07T15:15:58,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:58,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. as already flushing 2024-12-07T15:15:58,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:58,294 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] handler.RSProcedureHandler(58): pid=144 java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:58,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=144 java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:58,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=144 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:58,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:58,434 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:58,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:58,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43956 deadline: 1733584618433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:58,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1733584618434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:58,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43928 deadline: 1733584618434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:58,434 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:58,434 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:58,434 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:58,434 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:58,434 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:58,434 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:58,434 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:58,434 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:58,434 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:58,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:15:58,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43912 deadline: 1733584618443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:15:58,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:58,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:15:58,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., hostname=d6797ebe9c52,46009,1733584418401, seqNum=2 because the exception is null or not the one we care about 2024-12-07T15:15:58,446 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-12-07T15:15:58,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:58,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. as already flushing 2024-12-07T15:15:58,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:58,446 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] handler.RSProcedureHandler(58): pid=144 java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:58,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=144 java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:58,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=144 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:58,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-12-07T15:15:58,499 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/B/ea2efeee56f24ad2a4aaaed2ac9b404b 2024-12-07T15:15:58,504 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/C/5e0dda0b83ca4fc485303950380d09ef is 50, key is test_row_0/C:col10/1733584557678/Put/seqid=0 2024-12-07T15:15:58,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742348_1524 (size=12301) 2024-12-07T15:15:58,599 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-12-07T15:15:58,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:58,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. as already flushing 2024-12-07T15:15:58,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:58,599 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] handler.RSProcedureHandler(58): pid=144 java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:58,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=144 java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:58,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=144 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:58,750 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-12-07T15:15:58,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:58,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. as already flushing 2024-12-07T15:15:58,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:58,751 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] handler.RSProcedureHandler(58): pid=144 java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:58,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=144 java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:58,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=144 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:58,903 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-12-07T15:15:58,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:58,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. as already flushing 2024-12-07T15:15:58,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:58,904 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] handler.RSProcedureHandler(58): pid=144 java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:58,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=144 java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:58,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=144 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:58,907 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/C/5e0dda0b83ca4fc485303950380d09ef 2024-12-07T15:15:58,911 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/A/ff0bdcb4fa9b4d2d8803f798c79a8f1d as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/ff0bdcb4fa9b4d2d8803f798c79a8f1d 2024-12-07T15:15:58,913 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/ff0bdcb4fa9b4d2d8803f798c79a8f1d, entries=200, sequenceid=362, filesize=14.4 K 2024-12-07T15:15:58,914 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/B/ea2efeee56f24ad2a4aaaed2ac9b404b as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/ea2efeee56f24ad2a4aaaed2ac9b404b 2024-12-07T15:15:58,916 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/ea2efeee56f24ad2a4aaaed2ac9b404b, entries=150, sequenceid=362, filesize=12.0 K 2024-12-07T15:15:58,917 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/C/5e0dda0b83ca4fc485303950380d09ef as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/5e0dda0b83ca4fc485303950380d09ef 2024-12-07T15:15:58,919 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/5e0dda0b83ca4fc485303950380d09ef, entries=150, sequenceid=362, filesize=12.0 K 2024-12-07T15:15:58,920 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 54b89cbce0d0fe96d7c51211398eae76 in 1238ms, sequenceid=362, compaction requested=true 2024-12-07T15:15:58,920 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 54b89cbce0d0fe96d7c51211398eae76: 2024-12-07T15:15:58,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54b89cbce0d0fe96d7c51211398eae76:A, priority=-2147483648, current under compaction store size is 1 2024-12-07T15:15:58,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:15:58,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54b89cbce0d0fe96d7c51211398eae76:B, priority=-2147483648, current under compaction store size is 2 2024-12-07T15:15:58,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:15:58,920 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:15:58,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54b89cbce0d0fe96d7c51211398eae76:C, priority=-2147483648, current under compaction store size is 3 2024-12-07T15:15:58,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T15:15:58,920 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:15:58,921 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40195 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:15:58,921 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:15:58,921 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1541): 54b89cbce0d0fe96d7c51211398eae76/A is initiating minor compaction (all files) 2024-12-07T15:15:58,921 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1541): 54b89cbce0d0fe96d7c51211398eae76/B is initiating minor compaction (all files) 2024-12-07T15:15:58,921 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54b89cbce0d0fe96d7c51211398eae76/A in TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:58,921 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54b89cbce0d0fe96d7c51211398eae76/B in TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:58,921 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/92c28d5fd2c446c7aa0303def717e493, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/3d71f26c7fb34ed58b620b430d6910a6, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/ff0bdcb4fa9b4d2d8803f798c79a8f1d] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp, totalSize=39.3 K 2024-12-07T15:15:58,921 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/b2f45ce89b094fee899952aa129c8f34, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/2e27d42aea2d4124adf26c2a4e5e1d51, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/ea2efeee56f24ad2a4aaaed2ac9b404b] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp, totalSize=36.9 K 2024-12-07T15:15:58,921 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting b2f45ce89b094fee899952aa129c8f34, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1733584555278 2024-12-07T15:15:58,921 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 92c28d5fd2c446c7aa0303def717e493, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1733584555278 2024-12-07T15:15:58,921 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e27d42aea2d4124adf26c2a4e5e1d51, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1733584556488 2024-12-07T15:15:58,921 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d71f26c7fb34ed58b620b430d6910a6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1733584556488 2024-12-07T15:15:58,922 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting ea2efeee56f24ad2a4aaaed2ac9b404b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1733584557678 2024-12-07T15:15:58,922 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff0bdcb4fa9b4d2d8803f798c79a8f1d, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1733584557678 2024-12-07T15:15:58,927 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54b89cbce0d0fe96d7c51211398eae76#B#compaction#451 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:15:58,927 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54b89cbce0d0fe96d7c51211398eae76#A#compaction#452 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:15:58,927 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/B/2cf3851e2600435c84d9c3d66ae87710 is 50, key is test_row_0/B:col10/1733584557678/Put/seqid=0 2024-12-07T15:15:58,927 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/A/3e2129ae229b440d9fc2af86f64addc8 is 50, key is test_row_0/A:col10/1733584557678/Put/seqid=0 2024-12-07T15:15:58,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742349_1525 (size=13255) 2024-12-07T15:15:58,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742350_1526 (size=13255) 2024-12-07T15:15:58,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on 54b89cbce0d0fe96d7c51211398eae76 2024-12-07T15:15:58,945 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 54b89cbce0d0fe96d7c51211398eae76 3/3 column families, dataSize=154.31 KB heapSize=436.08 KB 2024-12-07T15:15:58,945 INFO [Thread-1977 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1977. 2024-12-07T15:15:58,945 INFO [Thread-1983 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1983. 2024-12-07T15:15:58,945 INFO [Thread-1979 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1979. 2024-12-07T15:15:58,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b89cbce0d0fe96d7c51211398eae76, store=A 2024-12-07T15:15:58,945 DEBUG [Thread-1977 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T15:15:58,945 DEBUG [Thread-1983 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T15:15:58,945 DEBUG [Thread-1979 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T15:15:58,945 DEBUG [Thread-1977 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:15:58,945 DEBUG [Thread-1983 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:15:58,945 DEBUG [Thread-1979 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:15:58,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-07T15:15:58,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b89cbce0d0fe96d7c51211398eae76, store=B 2024-12-07T15:15:58,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-07T15:15:58,945 DEBUG [Thread-1977 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:15:58,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b89cbce0d0fe96d7c51211398eae76, store=C 2024-12-07T15:15:58,945 DEBUG [Thread-1979 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:15:58,945 DEBUG [Thread-1983 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:15:58,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-07T15:15:58,946 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T15:15:58,946 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T15:15:58,946 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T15:15:58,949 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/A/1ba256f33c704086895a18d1cb35ebb8 is 50, key is test_row_0/A:col10/1733584558944/Put/seqid=0 2024-12-07T15:15:58,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742351_1527 (size=12301) 2024-12-07T15:15:58,954 INFO [Thread-1981 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1981. 2024-12-07T15:15:58,954 DEBUG [Thread-1981 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T15:15:58,954 DEBUG [Thread-1981 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:15:58,954 DEBUG [Thread-1981 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:15:58,954 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T15:15:58,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-12-07T15:15:59,056 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-12-07T15:15:59,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:59,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. as already flushing 2024-12-07T15:15:59,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:59,056 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] handler.RSProcedureHandler(58): pid=144 java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:59,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=144 java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:59,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=144 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:59,209 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-12-07T15:15:59,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:59,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. as already flushing 2024-12-07T15:15:59,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:59,209 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] handler.RSProcedureHandler(58): pid=144 java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:59,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=144 java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:59,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=144 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:59,334 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/A/3e2129ae229b440d9fc2af86f64addc8 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/3e2129ae229b440d9fc2af86f64addc8 2024-12-07T15:15:59,334 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/B/2cf3851e2600435c84d9c3d66ae87710 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/2cf3851e2600435c84d9c3d66ae87710 2024-12-07T15:15:59,337 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54b89cbce0d0fe96d7c51211398eae76/A of 54b89cbce0d0fe96d7c51211398eae76 into 3e2129ae229b440d9fc2af86f64addc8(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:15:59,337 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54b89cbce0d0fe96d7c51211398eae76/B of 54b89cbce0d0fe96d7c51211398eae76 into 2cf3851e2600435c84d9c3d66ae87710(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:15:59,337 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54b89cbce0d0fe96d7c51211398eae76: 2024-12-07T15:15:59,337 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54b89cbce0d0fe96d7c51211398eae76: 2024-12-07T15:15:59,337 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., storeName=54b89cbce0d0fe96d7c51211398eae76/A, priority=13, startTime=1733584558920; duration=0sec 2024-12-07T15:15:59,337 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., storeName=54b89cbce0d0fe96d7c51211398eae76/B, priority=13, startTime=1733584558920; duration=0sec 2024-12-07T15:15:59,337 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T15:15:59,337 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T15:15:59,337 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54b89cbce0d0fe96d7c51211398eae76:A 2024-12-07T15:15:59,337 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54b89cbce0d0fe96d7c51211398eae76:B 2024-12-07T15:15:59,337 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:15:59,338 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:15:59,338 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1541): 54b89cbce0d0fe96d7c51211398eae76/C is initiating minor compaction (all files) 2024-12-07T15:15:59,338 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54b89cbce0d0fe96d7c51211398eae76/C in TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:59,338 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/5db057dda44c4fb3ba05d6c5292e2036, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/4a4052a53ba14869b783409776075bb8, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/5e0dda0b83ca4fc485303950380d09ef] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp, totalSize=36.9 K 2024-12-07T15:15:59,338 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5db057dda44c4fb3ba05d6c5292e2036, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1733584555278 2024-12-07T15:15:59,339 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a4052a53ba14869b783409776075bb8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1733584556488 2024-12-07T15:15:59,339 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e0dda0b83ca4fc485303950380d09ef, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1733584557678 2024-12-07T15:15:59,344 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54b89cbce0d0fe96d7c51211398eae76#C#compaction#454 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:15:59,344 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/C/fdb0f76a700c40449f4d8983b780f7d5 is 50, key is test_row_0/C:col10/1733584557678/Put/seqid=0 2024-12-07T15:15:59,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742352_1528 (size=13255) 2024-12-07T15:15:59,352 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=401 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/A/1ba256f33c704086895a18d1cb35ebb8 2024-12-07T15:15:59,357 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/B/85fd5c4e425845439dce6b19475d697f is 50, key is test_row_0/B:col10/1733584558944/Put/seqid=0 2024-12-07T15:15:59,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742353_1529 (size=12301) 2024-12-07T15:15:59,361 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-12-07T15:15:59,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:59,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. as already flushing 2024-12-07T15:15:59,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:59,362 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] handler.RSProcedureHandler(58): pid=144 java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:59,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=144 java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:59,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=144 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:59,454 INFO [Thread-1975 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1975. 2024-12-07T15:15:59,454 DEBUG [Thread-1975 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T15:15:59,454 DEBUG [Thread-1975 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:15:59,455 DEBUG [Thread-1975 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:15:59,455 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T15:15:59,514 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-12-07T15:15:59,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:59,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. as already flushing 2024-12-07T15:15:59,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:59,514 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] handler.RSProcedureHandler(58): pid=144 java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:59,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=144 java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:59,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=144 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:59,666 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-12-07T15:15:59,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:59,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. as already flushing 2024-12-07T15:15:59,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:59,667 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] handler.RSProcedureHandler(58): pid=144 java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:59,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=144 java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:59,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=144 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:59,750 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/C/fdb0f76a700c40449f4d8983b780f7d5 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/fdb0f76a700c40449f4d8983b780f7d5 2024-12-07T15:15:59,754 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54b89cbce0d0fe96d7c51211398eae76/C of 54b89cbce0d0fe96d7c51211398eae76 into fdb0f76a700c40449f4d8983b780f7d5(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:15:59,754 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54b89cbce0d0fe96d7c51211398eae76: 2024-12-07T15:15:59,754 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76., storeName=54b89cbce0d0fe96d7c51211398eae76/C, priority=13, startTime=1733584558920; duration=0sec 2024-12-07T15:15:59,754 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:15:59,754 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54b89cbce0d0fe96d7c51211398eae76:C 2024-12-07T15:15:59,760 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=401 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/B/85fd5c4e425845439dce6b19475d697f 2024-12-07T15:15:59,765 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/C/d848df39931648c9b0f9cdf77fb4b9ae is 50, key is test_row_0/C:col10/1733584558944/Put/seqid=0 2024-12-07T15:15:59,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742354_1530 (size=12301) 2024-12-07T15:15:59,819 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-12-07T15:15:59,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:59,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. as already flushing 2024-12-07T15:15:59,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:59,819 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] handler.RSProcedureHandler(58): pid=144 java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:59,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=144 java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:59,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=144 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:59,972 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-12-07T15:15:59,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:59,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. as already flushing 2024-12-07T15:15:59,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:15:59,972 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] handler.RSProcedureHandler(58): pid=144 java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:59,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=144 java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:59,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=144 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:15:59,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-12-07T15:16:00,125 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-12-07T15:16:00,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:16:00,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. as already flushing 2024-12-07T15:16:00,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:16:00,125 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] handler.RSProcedureHandler(58): pid=144 java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:16:00,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=144 java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:16:00,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=144 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:16:00,169 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=401 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/C/d848df39931648c9b0f9cdf77fb4b9ae 2024-12-07T15:16:00,172 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/A/1ba256f33c704086895a18d1cb35ebb8 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/1ba256f33c704086895a18d1cb35ebb8 2024-12-07T15:16:00,174 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/1ba256f33c704086895a18d1cb35ebb8, entries=150, sequenceid=401, filesize=12.0 K 2024-12-07T15:16:00,175 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/B/85fd5c4e425845439dce6b19475d697f as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/85fd5c4e425845439dce6b19475d697f 2024-12-07T15:16:00,177 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/85fd5c4e425845439dce6b19475d697f, entries=150, sequenceid=401, filesize=12.0 K 2024-12-07T15:16:00,178 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/C/d848df39931648c9b0f9cdf77fb4b9ae as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/d848df39931648c9b0f9cdf77fb4b9ae 2024-12-07T15:16:00,180 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/d848df39931648c9b0f9cdf77fb4b9ae, entries=150, sequenceid=401, filesize=12.0 K 2024-12-07T15:16:00,181 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~436.03 KB/446496, currentSize=13.42 KB/13740 for 54b89cbce0d0fe96d7c51211398eae76 in 1236ms, sequenceid=401, compaction requested=false 2024-12-07T15:16:00,181 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 54b89cbce0d0fe96d7c51211398eae76: 2024-12-07T15:16:00,278 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-12-07T15:16:00,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:16:00,278 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2902): Flushing 54b89cbce0d0fe96d7c51211398eae76 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-12-07T15:16:00,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b89cbce0d0fe96d7c51211398eae76, store=A 2024-12-07T15:16:00,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:16:00,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b89cbce0d0fe96d7c51211398eae76, store=B 2024-12-07T15:16:00,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:16:00,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b89cbce0d0fe96d7c51211398eae76, store=C 2024-12-07T15:16:00,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:16:00,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/A/0c44f0e2fe4146b4844ea8adf117baaf is 50, key is test_row_0/A:col10/1733584558953/Put/seqid=0 2024-12-07T15:16:00,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742355_1531 (size=9857) 2024-12-07T15:16:00,685 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/A/0c44f0e2fe4146b4844ea8adf117baaf 2024-12-07T15:16:00,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/B/c68d1fe8a8e84554a86cd8897c12d98f is 50, key is test_row_0/B:col10/1733584558953/Put/seqid=0 2024-12-07T15:16:00,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742356_1532 (size=9857) 2024-12-07T15:16:01,093 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/B/c68d1fe8a8e84554a86cd8897c12d98f 2024-12-07T15:16:01,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/C/d1862310f6144b6a9372006b8afa4ff6 is 50, key is test_row_0/C:col10/1733584558953/Put/seqid=0 2024-12-07T15:16:01,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742357_1533 (size=9857) 2024-12-07T15:16:01,502 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/C/d1862310f6144b6a9372006b8afa4ff6 2024-12-07T15:16:01,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/A/0c44f0e2fe4146b4844ea8adf117baaf as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/0c44f0e2fe4146b4844ea8adf117baaf 2024-12-07T15:16:01,507 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/0c44f0e2fe4146b4844ea8adf117baaf, entries=100, sequenceid=409, filesize=9.6 K 2024-12-07T15:16:01,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/B/c68d1fe8a8e84554a86cd8897c12d98f as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/c68d1fe8a8e84554a86cd8897c12d98f 2024-12-07T15:16:01,510 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/c68d1fe8a8e84554a86cd8897c12d98f, entries=100, sequenceid=409, filesize=9.6 K 2024-12-07T15:16:01,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/.tmp/C/d1862310f6144b6a9372006b8afa4ff6 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/d1862310f6144b6a9372006b8afa4ff6 2024-12-07T15:16:01,513 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/d1862310f6144b6a9372006b8afa4ff6, entries=100, sequenceid=409, filesize=9.6 K 2024-12-07T15:16:01,513 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 54b89cbce0d0fe96d7c51211398eae76 in 1235ms, sequenceid=409, compaction requested=true 2024-12-07T15:16:01,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2603): Flush status journal for 54b89cbce0d0fe96d7c51211398eae76: 2024-12-07T15:16:01,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:16:01,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=144 2024-12-07T15:16:01,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster(4169): Remote procedure done, pid=144 2024-12-07T15:16:01,516 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-12-07T15:16:01,516 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.6790 sec 2024-12-07T15:16:01,517 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees in 3.6830 sec 2024-12-07T15:16:02,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-12-07T15:16:02,003 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T15:16:02,003 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-07T15:16:02,003 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 60 2024-12-07T15:16:02,003 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 68 2024-12-07T15:16:02,003 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 61 2024-12-07T15:16:02,003 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 63 2024-12-07T15:16:02,003 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 62 2024-12-07T15:16:02,003 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-07T15:16:02,003 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-07T15:16:02,003 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1154 2024-12-07T15:16:02,003 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3459 rows 2024-12-07T15:16:02,003 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1148 2024-12-07T15:16:02,003 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3441 rows 2024-12-07T15:16:02,003 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1148 2024-12-07T15:16:02,003 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3444 rows 2024-12-07T15:16:02,003 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1149 2024-12-07T15:16:02,003 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3444 rows 2024-12-07T15:16:02,003 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1155 2024-12-07T15:16:02,003 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3465 rows 2024-12-07T15:16:02,003 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-07T15:16:02,003 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-07T15:16:02,004 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:16:02,004 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:16:02,004 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T15:16:02,004 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-07T15:16:02,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] procedure2.ProcedureExecutor(1139): Stored pid=145, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-07T15:16:02,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-12-07T15:16:02,007 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733584562007"}]},"ts":"1733584562007"} 2024-12-07T15:16:02,008 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-07T15:16:02,008 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-07T15:16:02,009 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-07T15:16:02,010 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=54b89cbce0d0fe96d7c51211398eae76, UNASSIGN}] 2024-12-07T15:16:02,010 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=54b89cbce0d0fe96d7c51211398eae76, UNASSIGN 2024-12-07T15:16:02,011 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=54b89cbce0d0fe96d7c51211398eae76, regionState=CLOSING, regionLocation=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:02,012 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=54b89cbce0d0fe96d7c51211398eae76, UNASSIGN because future has completed 2024-12-07T15:16:02,012 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-07T15:16:02,012 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE, hasLock=false; CloseRegionProcedure 54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401}] 2024-12-07T15:16:02,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-12-07T15:16:02,164 INFO [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(122): Close 54b89cbce0d0fe96d7c51211398eae76 2024-12-07T15:16:02,164 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-07T15:16:02,164 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1722): Closing 54b89cbce0d0fe96d7c51211398eae76, disabling compactions & flushes 2024-12-07T15:16:02,165 INFO [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:16:02,165 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:16:02,165 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. after waiting 0 ms 2024-12-07T15:16:02,165 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:16:02,165 DEBUG [StoreCloser-TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/dabb52f8fc0e40d68fb52038c6b5d63d, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/923f834ced3840c1b0756b4749e64a80, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/5646c131a0dc44338639daf197194ef7, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/d1eb582c009d48f2a0ac18eca652e901, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/1634e8062e774003a6c355d3f2747ecf, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/176d80eb058c4138849e38eb9b6d9698, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/6c20ccda6d184c62b3ffe21d39b56d97, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/a5cf5ec83742473594991e6ed0e89f4b, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/deb6b4d4c46b4c65baa2ebb7da18a213, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/bdccd8c0a164446ba1c2785f6bfc9aa6, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/1b6d311ed3964ec9a446855f29699236, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/2e46a5a69b754281ba17e10078630f06, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/36616ec865804a70b75a5f6c2d6439da, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/48b2da53004b490a955cde74ae017810, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/887696742482454da8a3be6344ec1aea, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/38f20619c1954ee9b29909eb2509766b, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/ec560318fd3c4a16bd05cef580798755, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/ad3cd1aaf0f04fed8470bbb5cd364179, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/c2c30357eec34a4db091e67c5afdf270, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/14c2c642404545f7bd1bda37dd0a7b39, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/e0eca1148ef443568bc369ba7a5faf7b, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/88493b2dcb7543708a62fb48225d5960, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/bd0e603fc2f0444391957559010e18a3, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/e8b0f3fad0224516a056945dd0f77b73, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/37af6e60eebf460e925e7ad2fbe0fbf6, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/92c28d5fd2c446c7aa0303def717e493, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/3d71f26c7fb34ed58b620b430d6910a6, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/ff0bdcb4fa9b4d2d8803f798c79a8f1d] to archive 2024-12-07T15:16:02,166 DEBUG [StoreCloser-TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-07T15:16:02,167 DEBUG [StoreCloser-TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/dabb52f8fc0e40d68fb52038c6b5d63d to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/dabb52f8fc0e40d68fb52038c6b5d63d 2024-12-07T15:16:02,168 DEBUG [StoreCloser-TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/923f834ced3840c1b0756b4749e64a80 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/923f834ced3840c1b0756b4749e64a80 2024-12-07T15:16:02,169 DEBUG [StoreCloser-TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/5646c131a0dc44338639daf197194ef7 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/5646c131a0dc44338639daf197194ef7 2024-12-07T15:16:02,170 DEBUG [StoreCloser-TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/d1eb582c009d48f2a0ac18eca652e901 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/d1eb582c009d48f2a0ac18eca652e901 2024-12-07T15:16:02,170 DEBUG [StoreCloser-TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/1634e8062e774003a6c355d3f2747ecf to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/1634e8062e774003a6c355d3f2747ecf 2024-12-07T15:16:02,171 DEBUG [StoreCloser-TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/176d80eb058c4138849e38eb9b6d9698 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/176d80eb058c4138849e38eb9b6d9698 2024-12-07T15:16:02,172 DEBUG [StoreCloser-TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/6c20ccda6d184c62b3ffe21d39b56d97 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/6c20ccda6d184c62b3ffe21d39b56d97 2024-12-07T15:16:02,173 DEBUG [StoreCloser-TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/a5cf5ec83742473594991e6ed0e89f4b to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/a5cf5ec83742473594991e6ed0e89f4b 2024-12-07T15:16:02,173 DEBUG [StoreCloser-TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/deb6b4d4c46b4c65baa2ebb7da18a213 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/deb6b4d4c46b4c65baa2ebb7da18a213 2024-12-07T15:16:02,174 DEBUG [StoreCloser-TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/bdccd8c0a164446ba1c2785f6bfc9aa6 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/bdccd8c0a164446ba1c2785f6bfc9aa6 2024-12-07T15:16:02,175 DEBUG [StoreCloser-TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/1b6d311ed3964ec9a446855f29699236 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/1b6d311ed3964ec9a446855f29699236 2024-12-07T15:16:02,175 DEBUG [StoreCloser-TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/2e46a5a69b754281ba17e10078630f06 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/2e46a5a69b754281ba17e10078630f06 2024-12-07T15:16:02,176 DEBUG [StoreCloser-TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/36616ec865804a70b75a5f6c2d6439da to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/36616ec865804a70b75a5f6c2d6439da 2024-12-07T15:16:02,177 DEBUG [StoreCloser-TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/48b2da53004b490a955cde74ae017810 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/48b2da53004b490a955cde74ae017810 2024-12-07T15:16:02,178 DEBUG [StoreCloser-TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/887696742482454da8a3be6344ec1aea to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/887696742482454da8a3be6344ec1aea 2024-12-07T15:16:02,178 DEBUG [StoreCloser-TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/38f20619c1954ee9b29909eb2509766b to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/38f20619c1954ee9b29909eb2509766b 2024-12-07T15:16:02,179 DEBUG [StoreCloser-TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/ec560318fd3c4a16bd05cef580798755 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/ec560318fd3c4a16bd05cef580798755 2024-12-07T15:16:02,180 DEBUG [StoreCloser-TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/ad3cd1aaf0f04fed8470bbb5cd364179 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/ad3cd1aaf0f04fed8470bbb5cd364179 2024-12-07T15:16:02,181 DEBUG [StoreCloser-TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/c2c30357eec34a4db091e67c5afdf270 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/c2c30357eec34a4db091e67c5afdf270 2024-12-07T15:16:02,182 DEBUG [StoreCloser-TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/14c2c642404545f7bd1bda37dd0a7b39 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/14c2c642404545f7bd1bda37dd0a7b39 2024-12-07T15:16:02,183 DEBUG [StoreCloser-TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/e0eca1148ef443568bc369ba7a5faf7b to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/e0eca1148ef443568bc369ba7a5faf7b 2024-12-07T15:16:02,184 DEBUG [StoreCloser-TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/88493b2dcb7543708a62fb48225d5960 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/88493b2dcb7543708a62fb48225d5960 2024-12-07T15:16:02,185 DEBUG [StoreCloser-TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/bd0e603fc2f0444391957559010e18a3 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/bd0e603fc2f0444391957559010e18a3 2024-12-07T15:16:02,186 DEBUG [StoreCloser-TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/e8b0f3fad0224516a056945dd0f77b73 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/e8b0f3fad0224516a056945dd0f77b73 2024-12-07T15:16:02,187 DEBUG [StoreCloser-TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/37af6e60eebf460e925e7ad2fbe0fbf6 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/37af6e60eebf460e925e7ad2fbe0fbf6 2024-12-07T15:16:02,188 DEBUG [StoreCloser-TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/92c28d5fd2c446c7aa0303def717e493 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/92c28d5fd2c446c7aa0303def717e493 2024-12-07T15:16:02,188 DEBUG [StoreCloser-TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/3d71f26c7fb34ed58b620b430d6910a6 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/3d71f26c7fb34ed58b620b430d6910a6 2024-12-07T15:16:02,189 DEBUG [StoreCloser-TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/ff0bdcb4fa9b4d2d8803f798c79a8f1d to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/ff0bdcb4fa9b4d2d8803f798c79a8f1d 2024-12-07T15:16:02,190 DEBUG [StoreCloser-TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/56a77bcc7b8c49f2b0f0ab4e5209b3fc, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/ceeed6b9995a49efbc7b8fa54be5c384, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/6578588c544142e282d7c3001c6eebf2, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/70042830246648acae6493cc1efdd302, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/4749a1e92688422480c5513df39eec29, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/ff8575e113df49d89e98792faf4f01aa, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/6dcc0c54c6db447b8a652852eb319469, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/ccb1ba5153064582ad94672f1b720ef1, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/9392f77ed7bf40afacb77e32371616a1, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/cb76d07c6e9d465aaf6127771dfa97c9, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/dd8eb0296fe64e81997a690a0d0f93fd, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/61f305e26f974823bf5df076dbd6dc51, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/45d010b62f8b48aeb0612bbb3878976b, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/1b7478547f9c45cdac97dc5fbd7ade80, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/d7d5dc8b80924c3d8475be0fd3219987, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/e263ea4699c6403682fc9f5f798b6e27, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/7b3ac480e77540de94e578a3ed0a6a07, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/7f9b7eba129a47acbb98f9b7401c3539, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/6e79476335244b50be7af4f5aaa4a03e, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/5f676a18ea6b4aedb984bd8d1ec08e1d, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/1034eff85b2641baaf0e7629baeb7e98, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/3441a348dfc04f99ad62cc736dd51245, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/c236134de5cd47e289708b41b314a2b5, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/b1c1965820794d8e93f5de0d6a08a958, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/b2f45ce89b094fee899952aa129c8f34, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/526c01506d2141f49e8ced2ab4205320, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/2e27d42aea2d4124adf26c2a4e5e1d51, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/ea2efeee56f24ad2a4aaaed2ac9b404b] to archive 2024-12-07T15:16:02,191 DEBUG [StoreCloser-TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-07T15:16:02,192 DEBUG [StoreCloser-TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/56a77bcc7b8c49f2b0f0ab4e5209b3fc to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/56a77bcc7b8c49f2b0f0ab4e5209b3fc 2024-12-07T15:16:02,193 DEBUG [StoreCloser-TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/ceeed6b9995a49efbc7b8fa54be5c384 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/ceeed6b9995a49efbc7b8fa54be5c384 2024-12-07T15:16:02,194 DEBUG [StoreCloser-TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/6578588c544142e282d7c3001c6eebf2 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/6578588c544142e282d7c3001c6eebf2 2024-12-07T15:16:02,195 DEBUG [StoreCloser-TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/70042830246648acae6493cc1efdd302 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/70042830246648acae6493cc1efdd302 2024-12-07T15:16:02,195 DEBUG [StoreCloser-TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/4749a1e92688422480c5513df39eec29 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/4749a1e92688422480c5513df39eec29 2024-12-07T15:16:02,196 DEBUG [StoreCloser-TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/ff8575e113df49d89e98792faf4f01aa to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/ff8575e113df49d89e98792faf4f01aa 2024-12-07T15:16:02,197 DEBUG [StoreCloser-TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/6dcc0c54c6db447b8a652852eb319469 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/6dcc0c54c6db447b8a652852eb319469 2024-12-07T15:16:02,198 DEBUG [StoreCloser-TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/ccb1ba5153064582ad94672f1b720ef1 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/ccb1ba5153064582ad94672f1b720ef1 2024-12-07T15:16:02,199 DEBUG [StoreCloser-TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/9392f77ed7bf40afacb77e32371616a1 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/9392f77ed7bf40afacb77e32371616a1 2024-12-07T15:16:02,200 DEBUG [StoreCloser-TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/cb76d07c6e9d465aaf6127771dfa97c9 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/cb76d07c6e9d465aaf6127771dfa97c9 2024-12-07T15:16:02,201 DEBUG [StoreCloser-TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/dd8eb0296fe64e81997a690a0d0f93fd to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/dd8eb0296fe64e81997a690a0d0f93fd 2024-12-07T15:16:02,202 DEBUG [StoreCloser-TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/61f305e26f974823bf5df076dbd6dc51 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/61f305e26f974823bf5df076dbd6dc51 2024-12-07T15:16:02,202 DEBUG [StoreCloser-TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/45d010b62f8b48aeb0612bbb3878976b to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/45d010b62f8b48aeb0612bbb3878976b 2024-12-07T15:16:02,203 DEBUG [StoreCloser-TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/1b7478547f9c45cdac97dc5fbd7ade80 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/1b7478547f9c45cdac97dc5fbd7ade80 2024-12-07T15:16:02,204 DEBUG [StoreCloser-TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/d7d5dc8b80924c3d8475be0fd3219987 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/d7d5dc8b80924c3d8475be0fd3219987 2024-12-07T15:16:02,205 DEBUG [StoreCloser-TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/e263ea4699c6403682fc9f5f798b6e27 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/e263ea4699c6403682fc9f5f798b6e27 2024-12-07T15:16:02,206 DEBUG [StoreCloser-TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/7b3ac480e77540de94e578a3ed0a6a07 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/7b3ac480e77540de94e578a3ed0a6a07 2024-12-07T15:16:02,207 DEBUG [StoreCloser-TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/7f9b7eba129a47acbb98f9b7401c3539 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/7f9b7eba129a47acbb98f9b7401c3539 2024-12-07T15:16:02,208 DEBUG [StoreCloser-TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/6e79476335244b50be7af4f5aaa4a03e to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/6e79476335244b50be7af4f5aaa4a03e 2024-12-07T15:16:02,209 DEBUG [StoreCloser-TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/5f676a18ea6b4aedb984bd8d1ec08e1d to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/5f676a18ea6b4aedb984bd8d1ec08e1d 2024-12-07T15:16:02,210 DEBUG [StoreCloser-TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/1034eff85b2641baaf0e7629baeb7e98 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/1034eff85b2641baaf0e7629baeb7e98 2024-12-07T15:16:02,211 DEBUG [StoreCloser-TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/3441a348dfc04f99ad62cc736dd51245 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/3441a348dfc04f99ad62cc736dd51245 2024-12-07T15:16:02,212 DEBUG [StoreCloser-TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/c236134de5cd47e289708b41b314a2b5 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/c236134de5cd47e289708b41b314a2b5 2024-12-07T15:16:02,212 DEBUG [StoreCloser-TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/b1c1965820794d8e93f5de0d6a08a958 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/b1c1965820794d8e93f5de0d6a08a958 2024-12-07T15:16:02,213 DEBUG [StoreCloser-TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/b2f45ce89b094fee899952aa129c8f34 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/b2f45ce89b094fee899952aa129c8f34 2024-12-07T15:16:02,214 DEBUG [StoreCloser-TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/526c01506d2141f49e8ced2ab4205320 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/526c01506d2141f49e8ced2ab4205320 2024-12-07T15:16:02,214 DEBUG [StoreCloser-TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/2e27d42aea2d4124adf26c2a4e5e1d51 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/2e27d42aea2d4124adf26c2a4e5e1d51 2024-12-07T15:16:02,215 DEBUG [StoreCloser-TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/ea2efeee56f24ad2a4aaaed2ac9b404b to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/ea2efeee56f24ad2a4aaaed2ac9b404b 2024-12-07T15:16:02,216 DEBUG [StoreCloser-TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/ba0de81359b348508b24c616eb8dd827, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/f3998da92c3742729539b9d54ee959bb, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/a7474607a4ed4bbda251d1c0639795fb, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/f941a5cebd264600973c0c29a38c8c87, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/57a4deeb863c4fe58ee0cbe3fea400ef, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/de53b8864ad944a1bb39c0f7830cf6d0, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/eca82b3a850549a99957e609ebd19d85, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/3a06ff06cc7c4b5c81e43799748b6a7c, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/c3c5f1947f2d4cd9ba452be91a792b3f, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/6d440737de324e1690c47da1ef80d2e7, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/e66c14fa49e74bd399ab01cffb0e1e0f, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/7f9d90f6e337469d952001046a6d38ee, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/6f9d4549067f47b09c526f9732ba914e, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/e89b5c3bcb7d4d2fb2b5c7c3c8fc5749, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/b9c3b1ec8fd148909f4062a333b89910, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/e4afea5931d646f79c50a1ca325ce115, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/fb6b6c32d8934e7da62aab69e7f6ba5f, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/34831ea61f9f4084808d416b314b6ad1, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/6d2d9b0d599e422f8d01389b26f002fe, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/95e2a3008942455b9f9c04ca585ddf4d, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/74981f233a8d41f5b1ab4ad26316b7da, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/049ea8d678b944eeb16963628d8e44f0, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/2a6a9018366149bbb051746a4cdd1b99, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/533df453e2ed454f8a5403fc5e30ab87, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/5db057dda44c4fb3ba05d6c5292e2036, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/7af8bb18aaca42aabea4f6eaa66a2805, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/4a4052a53ba14869b783409776075bb8, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/5e0dda0b83ca4fc485303950380d09ef] to archive 2024-12-07T15:16:02,217 DEBUG [StoreCloser-TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-07T15:16:02,218 DEBUG [StoreCloser-TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/ba0de81359b348508b24c616eb8dd827 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/ba0de81359b348508b24c616eb8dd827 2024-12-07T15:16:02,218 DEBUG [StoreCloser-TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/f3998da92c3742729539b9d54ee959bb to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/f3998da92c3742729539b9d54ee959bb 2024-12-07T15:16:02,219 DEBUG [StoreCloser-TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/a7474607a4ed4bbda251d1c0639795fb to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/a7474607a4ed4bbda251d1c0639795fb 2024-12-07T15:16:02,220 DEBUG [StoreCloser-TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/f941a5cebd264600973c0c29a38c8c87 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/f941a5cebd264600973c0c29a38c8c87 2024-12-07T15:16:02,221 DEBUG [StoreCloser-TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/57a4deeb863c4fe58ee0cbe3fea400ef to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/57a4deeb863c4fe58ee0cbe3fea400ef 2024-12-07T15:16:02,221 DEBUG [StoreCloser-TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/de53b8864ad944a1bb39c0f7830cf6d0 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/de53b8864ad944a1bb39c0f7830cf6d0 2024-12-07T15:16:02,222 DEBUG [StoreCloser-TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/eca82b3a850549a99957e609ebd19d85 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/eca82b3a850549a99957e609ebd19d85 2024-12-07T15:16:02,223 DEBUG [StoreCloser-TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/3a06ff06cc7c4b5c81e43799748b6a7c to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/3a06ff06cc7c4b5c81e43799748b6a7c 2024-12-07T15:16:02,223 DEBUG [StoreCloser-TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/c3c5f1947f2d4cd9ba452be91a792b3f to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/c3c5f1947f2d4cd9ba452be91a792b3f 2024-12-07T15:16:02,224 DEBUG [StoreCloser-TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/6d440737de324e1690c47da1ef80d2e7 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/6d440737de324e1690c47da1ef80d2e7 2024-12-07T15:16:02,225 DEBUG [StoreCloser-TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/e66c14fa49e74bd399ab01cffb0e1e0f to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/e66c14fa49e74bd399ab01cffb0e1e0f 2024-12-07T15:16:02,225 DEBUG [StoreCloser-TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/7f9d90f6e337469d952001046a6d38ee to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/7f9d90f6e337469d952001046a6d38ee 2024-12-07T15:16:02,226 DEBUG [StoreCloser-TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/6f9d4549067f47b09c526f9732ba914e to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/6f9d4549067f47b09c526f9732ba914e 2024-12-07T15:16:02,226 DEBUG [StoreCloser-TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/e89b5c3bcb7d4d2fb2b5c7c3c8fc5749 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/e89b5c3bcb7d4d2fb2b5c7c3c8fc5749 2024-12-07T15:16:02,227 DEBUG [StoreCloser-TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/b9c3b1ec8fd148909f4062a333b89910 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/b9c3b1ec8fd148909f4062a333b89910 2024-12-07T15:16:02,228 DEBUG [StoreCloser-TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/e4afea5931d646f79c50a1ca325ce115 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/e4afea5931d646f79c50a1ca325ce115 2024-12-07T15:16:02,228 DEBUG [StoreCloser-TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/fb6b6c32d8934e7da62aab69e7f6ba5f to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/fb6b6c32d8934e7da62aab69e7f6ba5f 2024-12-07T15:16:02,229 DEBUG [StoreCloser-TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/34831ea61f9f4084808d416b314b6ad1 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/34831ea61f9f4084808d416b314b6ad1 2024-12-07T15:16:02,230 DEBUG [StoreCloser-TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/6d2d9b0d599e422f8d01389b26f002fe to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/6d2d9b0d599e422f8d01389b26f002fe 2024-12-07T15:16:02,230 DEBUG [StoreCloser-TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/95e2a3008942455b9f9c04ca585ddf4d to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/95e2a3008942455b9f9c04ca585ddf4d 2024-12-07T15:16:02,231 DEBUG [StoreCloser-TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/74981f233a8d41f5b1ab4ad26316b7da to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/74981f233a8d41f5b1ab4ad26316b7da 2024-12-07T15:16:02,231 DEBUG [StoreCloser-TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/049ea8d678b944eeb16963628d8e44f0 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/049ea8d678b944eeb16963628d8e44f0 2024-12-07T15:16:02,232 DEBUG [StoreCloser-TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/2a6a9018366149bbb051746a4cdd1b99 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/2a6a9018366149bbb051746a4cdd1b99 2024-12-07T15:16:02,233 DEBUG [StoreCloser-TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/533df453e2ed454f8a5403fc5e30ab87 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/533df453e2ed454f8a5403fc5e30ab87 2024-12-07T15:16:02,233 DEBUG [StoreCloser-TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/5db057dda44c4fb3ba05d6c5292e2036 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/5db057dda44c4fb3ba05d6c5292e2036 2024-12-07T15:16:02,234 DEBUG [StoreCloser-TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/7af8bb18aaca42aabea4f6eaa66a2805 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/7af8bb18aaca42aabea4f6eaa66a2805 2024-12-07T15:16:02,235 DEBUG [StoreCloser-TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/4a4052a53ba14869b783409776075bb8 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/4a4052a53ba14869b783409776075bb8 2024-12-07T15:16:02,236 DEBUG [StoreCloser-TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/5e0dda0b83ca4fc485303950380d09ef to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/5e0dda0b83ca4fc485303950380d09ef 2024-12-07T15:16:02,239 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/recovered.edits/412.seqid, newMaxSeqId=412, maxSeqId=1 2024-12-07T15:16:02,240 INFO [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76. 2024-12-07T15:16:02,240 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1676): Region close journal for 54b89cbce0d0fe96d7c51211398eae76: Waiting for close lock at 1733584562164Running coprocessor pre-close hooks at 1733584562164Disabling compacts and flushes for region at 1733584562164Disabling writes for close at 1733584562165 (+1 ms)Writing region close event to WAL at 1733584562237 (+72 ms)Running coprocessor post-close hooks at 1733584562240 (+3 ms)Closed at 1733584562240 2024-12-07T15:16:02,241 INFO [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(157): Closed 54b89cbce0d0fe96d7c51211398eae76 2024-12-07T15:16:02,242 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=54b89cbce0d0fe96d7c51211398eae76, regionState=CLOSED 2024-12-07T15:16:02,243 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=148, ppid=147, state=RUNNABLE, hasLock=false; CloseRegionProcedure 54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 because future has completed 2024-12-07T15:16:02,245 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-12-07T15:16:02,245 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; CloseRegionProcedure 54b89cbce0d0fe96d7c51211398eae76, server=d6797ebe9c52,46009,1733584418401 in 232 msec 2024-12-07T15:16:02,246 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-12-07T15:16:02,246 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=54b89cbce0d0fe96d7c51211398eae76, UNASSIGN in 235 msec 2024-12-07T15:16:02,248 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-12-07T15:16:02,248 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 238 msec 2024-12-07T15:16:02,249 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733584562249"}]},"ts":"1733584562249"} 2024-12-07T15:16:02,250 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-07T15:16:02,250 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-07T15:16:02,251 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 247 msec 2024-12-07T15:16:02,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-12-07T15:16:02,324 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-07T15:16:02,324 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-07T15:16:02,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] procedure2.ProcedureExecutor(1139): Stored pid=149, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-07T15:16:02,325 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=149, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-07T15:16:02,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=149 2024-12-07T15:16:02,326 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=149, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-07T15:16:02,328 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76 2024-12-07T15:16:02,330 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A, FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B, FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C, FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/recovered.edits] 2024-12-07T15:16:02,332 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/0c44f0e2fe4146b4844ea8adf117baaf to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/0c44f0e2fe4146b4844ea8adf117baaf 2024-12-07T15:16:02,332 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/1ba256f33c704086895a18d1cb35ebb8 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/1ba256f33c704086895a18d1cb35ebb8 2024-12-07T15:16:02,333 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/3e2129ae229b440d9fc2af86f64addc8 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/A/3e2129ae229b440d9fc2af86f64addc8 2024-12-07T15:16:02,335 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/2cf3851e2600435c84d9c3d66ae87710 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/2cf3851e2600435c84d9c3d66ae87710 2024-12-07T15:16:02,336 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/85fd5c4e425845439dce6b19475d697f to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/85fd5c4e425845439dce6b19475d697f 2024-12-07T15:16:02,337 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/c68d1fe8a8e84554a86cd8897c12d98f to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/B/c68d1fe8a8e84554a86cd8897c12d98f 2024-12-07T15:16:02,338 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/d1862310f6144b6a9372006b8afa4ff6 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/d1862310f6144b6a9372006b8afa4ff6 2024-12-07T15:16:02,339 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/d848df39931648c9b0f9cdf77fb4b9ae to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/d848df39931648c9b0f9cdf77fb4b9ae 2024-12-07T15:16:02,340 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/fdb0f76a700c40449f4d8983b780f7d5 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/C/fdb0f76a700c40449f4d8983b780f7d5 2024-12-07T15:16:02,342 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/recovered.edits/412.seqid to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76/recovered.edits/412.seqid 2024-12-07T15:16:02,342 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/54b89cbce0d0fe96d7c51211398eae76 2024-12-07T15:16:02,342 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-07T15:16:02,344 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=149, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-07T15:16:02,346 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-07T15:16:02,347 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-07T15:16:02,348 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=149, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-07T15:16:02,348 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-07T15:16:02,348 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733584562348"}]},"ts":"9223372036854775807"} 2024-12-07T15:16:02,350 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-07T15:16:02,350 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 54b89cbce0d0fe96d7c51211398eae76, NAME => 'TestAcidGuarantees,,1733584536750.54b89cbce0d0fe96d7c51211398eae76.', STARTKEY => '', ENDKEY => ''}] 2024-12-07T15:16:02,350 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-07T15:16:02,350 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733584562350"}]},"ts":"9223372036854775807"} 2024-12-07T15:16:02,351 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-07T15:16:02,352 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=149, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-07T15:16:02,353 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 28 msec 2024-12-07T15:16:02,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=149 2024-12-07T15:16:02,434 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-07T15:16:02,434 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-07T15:16:02,443 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=250 (was 247) - Thread LEAK? -, OpenFileDescriptor=449 (was 451), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=396 (was 438), ProcessCount=11 (was 11), AvailableMemoryMB=6747 (was 6859) 2024-12-07T15:16:02,451 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=250, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=396, ProcessCount=11, AvailableMemoryMB=6746 2024-12-07T15:16:02,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-07T15:16:02,453 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-07T15:16:02,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-07T15:16:02,455 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=150, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-07T15:16:02,455 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:16:02,455 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 150 2024-12-07T15:16:02,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-07T15:16:02,455 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=150, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-07T15:16:02,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742358_1534 (size=1061) 2024-12-07T15:16:02,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-07T15:16:02,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-07T15:16:02,861 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd 2024-12-07T15:16:02,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742359_1535 (size=53) 2024-12-07T15:16:03,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-07T15:16:03,266 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T15:16:03,266 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 069dbda53fbb065f68f88519d6f94bc4, disabling compactions & flushes 2024-12-07T15:16:03,266 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:03,266 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:03,266 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. after waiting 0 ms 2024-12-07T15:16:03,266 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:03,267 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:03,267 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 069dbda53fbb065f68f88519d6f94bc4: Waiting for close lock at 1733584563266Disabling compacts and flushes for region at 1733584563266Disabling writes for close at 1733584563266Writing region close event to WAL at 1733584563266Closed at 1733584563266 2024-12-07T15:16:03,268 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=150, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-07T15:16:03,268 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733584563268"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733584563268"}]},"ts":"1733584563268"} 2024-12-07T15:16:03,269 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-07T15:16:03,270 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=150, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-07T15:16:03,270 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733584563270"}]},"ts":"1733584563270"} 2024-12-07T15:16:03,271 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-07T15:16:03,271 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=069dbda53fbb065f68f88519d6f94bc4, ASSIGN}] 2024-12-07T15:16:03,272 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=069dbda53fbb065f68f88519d6f94bc4, ASSIGN 2024-12-07T15:16:03,272 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=069dbda53fbb065f68f88519d6f94bc4, ASSIGN; state=OFFLINE, location=d6797ebe9c52,46009,1733584418401; forceNewPlan=false, retain=false 2024-12-07T15:16:03,423 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=069dbda53fbb065f68f88519d6f94bc4, regionState=OPENING, regionLocation=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:03,424 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=069dbda53fbb065f68f88519d6f94bc4, ASSIGN because future has completed 2024-12-07T15:16:03,424 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=152, ppid=151, state=RUNNABLE, hasLock=false; OpenRegionProcedure 069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401}] 2024-12-07T15:16:03,578 INFO [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:03,578 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(7752): Opening region: {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} 2024-12-07T15:16:03,579 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 069dbda53fbb065f68f88519d6f94bc4 2024-12-07T15:16:03,579 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T15:16:03,579 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(7794): checking encryption for 069dbda53fbb065f68f88519d6f94bc4 2024-12-07T15:16:03,579 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(7797): checking classloading for 069dbda53fbb065f68f88519d6f94bc4 2024-12-07T15:16:03,580 INFO [StoreOpener-069dbda53fbb065f68f88519d6f94bc4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 069dbda53fbb065f68f88519d6f94bc4 2024-12-07T15:16:03,581 INFO [StoreOpener-069dbda53fbb065f68f88519d6f94bc4-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-07T15:16:03,581 INFO [StoreOpener-069dbda53fbb065f68f88519d6f94bc4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 069dbda53fbb065f68f88519d6f94bc4 columnFamilyName A 2024-12-07T15:16:03,581 DEBUG [StoreOpener-069dbda53fbb065f68f88519d6f94bc4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:16:03,581 INFO [StoreOpener-069dbda53fbb065f68f88519d6f94bc4-1 {}] regionserver.HStore(327): Store=069dbda53fbb065f68f88519d6f94bc4/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T15:16:03,581 INFO [StoreOpener-069dbda53fbb065f68f88519d6f94bc4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 069dbda53fbb065f68f88519d6f94bc4 2024-12-07T15:16:03,582 INFO [StoreOpener-069dbda53fbb065f68f88519d6f94bc4-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-07T15:16:03,582 INFO [StoreOpener-069dbda53fbb065f68f88519d6f94bc4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 069dbda53fbb065f68f88519d6f94bc4 columnFamilyName B 2024-12-07T15:16:03,582 DEBUG [StoreOpener-069dbda53fbb065f68f88519d6f94bc4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:16:03,583 INFO [StoreOpener-069dbda53fbb065f68f88519d6f94bc4-1 {}] regionserver.HStore(327): Store=069dbda53fbb065f68f88519d6f94bc4/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T15:16:03,583 INFO [StoreOpener-069dbda53fbb065f68f88519d6f94bc4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 069dbda53fbb065f68f88519d6f94bc4 2024-12-07T15:16:03,583 INFO [StoreOpener-069dbda53fbb065f68f88519d6f94bc4-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-07T15:16:03,584 INFO [StoreOpener-069dbda53fbb065f68f88519d6f94bc4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 069dbda53fbb065f68f88519d6f94bc4 columnFamilyName C 2024-12-07T15:16:03,584 DEBUG [StoreOpener-069dbda53fbb065f68f88519d6f94bc4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:16:03,584 INFO [StoreOpener-069dbda53fbb065f68f88519d6f94bc4-1 {}] regionserver.HStore(327): Store=069dbda53fbb065f68f88519d6f94bc4/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T15:16:03,584 INFO [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:03,584 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(1038): replaying wal for 069dbda53fbb065f68f88519d6f94bc4 2024-12-07T15:16:03,585 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4 2024-12-07T15:16:03,585 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4 2024-12-07T15:16:03,585 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(1048): stopping wal replay for 069dbda53fbb065f68f88519d6f94bc4 2024-12-07T15:16:03,586 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(1060): Cleaning up temporary data for 069dbda53fbb065f68f88519d6f94bc4 2024-12-07T15:16:03,586 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-07T15:16:03,587 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(1093): writing seq id for 069dbda53fbb065f68f88519d6f94bc4 2024-12-07T15:16:03,588 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T15:16:03,588 INFO [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(1114): Opened 069dbda53fbb065f68f88519d6f94bc4; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69412154, jitterRate=0.03432169556617737}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-07T15:16:03,588 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 069dbda53fbb065f68f88519d6f94bc4 2024-12-07T15:16:03,589 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(1006): Region open journal for 069dbda53fbb065f68f88519d6f94bc4: Running coprocessor pre-open hook at 1733584563579Writing region info on filesystem at 1733584563579Initializing all the Stores at 1733584563579Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733584563579Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733584563580 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733584563580Cleaning up temporary data from old regions at 1733584563586 (+6 ms)Running coprocessor post-open hooks at 1733584563588 (+2 ms)Region opened successfully at 1733584563589 (+1 ms) 2024-12-07T15:16:03,590 INFO [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., pid=152, masterSystemTime=1733584563576 2024-12-07T15:16:03,591 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:03,591 INFO [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:03,591 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=069dbda53fbb065f68f88519d6f94bc4, regionState=OPEN, openSeqNum=2, regionLocation=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:03,593 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=152, ppid=151, state=RUNNABLE, hasLock=false; OpenRegionProcedure 069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 because future has completed 2024-12-07T15:16:03,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-07T15:16:03,594 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=152, resume processing ppid=151 2024-12-07T15:16:03,594 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, ppid=151, state=SUCCESS, hasLock=false; OpenRegionProcedure 069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 in 169 msec 2024-12-07T15:16:03,596 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-12-07T15:16:03,596 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=069dbda53fbb065f68f88519d6f94bc4, ASSIGN in 323 msec 2024-12-07T15:16:03,596 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=150, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-07T15:16:03,596 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733584563596"}]},"ts":"1733584563596"} 2024-12-07T15:16:03,597 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-07T15:16:03,598 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=150, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-07T15:16:03,599 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1450 sec 2024-12-07T15:16:04,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-07T15:16:04,613 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-07T15:16:04,614 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66320673, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:16:04,614 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6797ebe9c52,36915,-1 for getting cluster id 2024-12-07T15:16:04,614 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T15:16:04,615 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f8956274-ed33-4e16-82eb-d5d5911ec893' 2024-12-07T15:16:04,615 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T15:16:04,615 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f8956274-ed33-4e16-82eb-d5d5911ec893" 2024-12-07T15:16:04,616 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36bd0eb1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:16:04,616 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6797ebe9c52,36915,-1] 2024-12-07T15:16:04,616 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T15:16:04,616 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:16:04,617 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53334, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T15:16:04,617 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f629534, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:16:04,617 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T15:16:04,618 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6797ebe9c52,46009,1733584418401, seqNum=-1] 2024-12-07T15:16:04,618 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T15:16:04,619 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44620, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T15:16:04,620 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-07T15:16:04,620 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is d6797ebe9c52,36915,1733584417704 2024-12-07T15:16:04,620 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@57aa64c1 2024-12-07T15:16:04,620 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-07T15:16:04,621 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53342, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-07T15:16:04,622 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-07T15:16:04,622 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-07T15:16:04,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] procedure2.ProcedureExecutor(1139): Stored pid=153, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-07T15:16:04,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742360_1536 (size=1097) 2024-12-07T15:16:05,032 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-07T15:16:05,032 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-07T15:16:05,034 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=154, ppid=153, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-07T15:16:05,036 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=069dbda53fbb065f68f88519d6f94bc4, REOPEN/MOVE}] 2024-12-07T15:16:05,037 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=155, ppid=154, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=069dbda53fbb065f68f88519d6f94bc4, REOPEN/MOVE 2024-12-07T15:16:05,037 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=155 updating hbase:meta row=069dbda53fbb065f68f88519d6f94bc4, regionState=CLOSING, regionLocation=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:05,038 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=155, ppid=154, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=069dbda53fbb065f68f88519d6f94bc4, REOPEN/MOVE because future has completed 2024-12-07T15:16:05,038 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-07T15:16:05,038 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=156, ppid=155, state=RUNNABLE, hasLock=false; CloseRegionProcedure 069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401}] 2024-12-07T15:16:05,191 INFO [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] handler.UnassignRegionHandler(122): Close 069dbda53fbb065f68f88519d6f94bc4 2024-12-07T15:16:05,191 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-07T15:16:05,191 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HRegion(1722): Closing 069dbda53fbb065f68f88519d6f94bc4, disabling compactions & flushes 2024-12-07T15:16:05,191 INFO [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:05,191 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:05,191 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. after waiting 0 ms 2024-12-07T15:16:05,191 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:05,194 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-07T15:16:05,195 INFO [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:05,195 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HRegion(1676): Region close journal for 069dbda53fbb065f68f88519d6f94bc4: Waiting for close lock at 1733584565191Running coprocessor pre-close hooks at 1733584565191Disabling compacts and flushes for region at 1733584565191Disabling writes for close at 1733584565191Writing region close event to WAL at 1733584565191Running coprocessor post-close hooks at 1733584565194 (+3 ms)Closed at 1733584565194 2024-12-07T15:16:05,195 WARN [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HRegionServer(3299): Not adding moved region record: 069dbda53fbb065f68f88519d6f94bc4 to self. 2024-12-07T15:16:05,196 INFO [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] handler.UnassignRegionHandler(157): Closed 069dbda53fbb065f68f88519d6f94bc4 2024-12-07T15:16:05,197 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=155 updating hbase:meta row=069dbda53fbb065f68f88519d6f94bc4, regionState=CLOSED 2024-12-07T15:16:05,198 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=156, ppid=155, state=RUNNABLE, hasLock=false; CloseRegionProcedure 069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 because future has completed 2024-12-07T15:16:05,200 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=156, resume processing ppid=155 2024-12-07T15:16:05,200 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, ppid=155, state=SUCCESS, hasLock=false; CloseRegionProcedure 069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 in 160 msec 2024-12-07T15:16:05,200 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=155, ppid=154, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=069dbda53fbb065f68f88519d6f94bc4, REOPEN/MOVE; state=CLOSED, location=d6797ebe9c52,46009,1733584418401; forceNewPlan=false, retain=true 2024-12-07T15:16:05,351 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=155 updating hbase:meta row=069dbda53fbb065f68f88519d6f94bc4, regionState=OPENING, regionLocation=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:05,352 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=155, ppid=154, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=069dbda53fbb065f68f88519d6f94bc4, REOPEN/MOVE because future has completed 2024-12-07T15:16:05,352 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=155, state=RUNNABLE, hasLock=false; OpenRegionProcedure 069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401}] 2024-12-07T15:16:05,506 INFO [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:05,506 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(7752): Opening region: {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} 2024-12-07T15:16:05,507 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 069dbda53fbb065f68f88519d6f94bc4 2024-12-07T15:16:05,507 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T15:16:05,507 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(7794): checking encryption for 069dbda53fbb065f68f88519d6f94bc4 2024-12-07T15:16:05,507 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(7797): checking classloading for 069dbda53fbb065f68f88519d6f94bc4 2024-12-07T15:16:05,508 INFO [StoreOpener-069dbda53fbb065f68f88519d6f94bc4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 069dbda53fbb065f68f88519d6f94bc4 2024-12-07T15:16:05,508 INFO [StoreOpener-069dbda53fbb065f68f88519d6f94bc4-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-07T15:16:05,509 INFO [StoreOpener-069dbda53fbb065f68f88519d6f94bc4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 069dbda53fbb065f68f88519d6f94bc4 columnFamilyName A 2024-12-07T15:16:05,509 DEBUG [StoreOpener-069dbda53fbb065f68f88519d6f94bc4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:16:05,510 INFO [StoreOpener-069dbda53fbb065f68f88519d6f94bc4-1 {}] regionserver.HStore(327): Store=069dbda53fbb065f68f88519d6f94bc4/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T15:16:05,510 INFO [StoreOpener-069dbda53fbb065f68f88519d6f94bc4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 069dbda53fbb065f68f88519d6f94bc4 2024-12-07T15:16:05,510 INFO [StoreOpener-069dbda53fbb065f68f88519d6f94bc4-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-07T15:16:05,510 INFO [StoreOpener-069dbda53fbb065f68f88519d6f94bc4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 069dbda53fbb065f68f88519d6f94bc4 columnFamilyName B 2024-12-07T15:16:05,511 DEBUG [StoreOpener-069dbda53fbb065f68f88519d6f94bc4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:16:05,511 INFO [StoreOpener-069dbda53fbb065f68f88519d6f94bc4-1 {}] regionserver.HStore(327): Store=069dbda53fbb065f68f88519d6f94bc4/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T15:16:05,511 INFO [StoreOpener-069dbda53fbb065f68f88519d6f94bc4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 069dbda53fbb065f68f88519d6f94bc4 2024-12-07T15:16:05,511 INFO [StoreOpener-069dbda53fbb065f68f88519d6f94bc4-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-07T15:16:05,511 INFO [StoreOpener-069dbda53fbb065f68f88519d6f94bc4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 069dbda53fbb065f68f88519d6f94bc4 columnFamilyName C 2024-12-07T15:16:05,511 DEBUG [StoreOpener-069dbda53fbb065f68f88519d6f94bc4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:16:05,512 INFO [StoreOpener-069dbda53fbb065f68f88519d6f94bc4-1 {}] regionserver.HStore(327): Store=069dbda53fbb065f68f88519d6f94bc4/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T15:16:05,512 INFO [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:05,512 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(1038): replaying wal for 069dbda53fbb065f68f88519d6f94bc4 2024-12-07T15:16:05,512 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4 2024-12-07T15:16:05,513 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4 2024-12-07T15:16:05,514 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(1048): stopping wal replay for 069dbda53fbb065f68f88519d6f94bc4 2024-12-07T15:16:05,514 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(1060): Cleaning up temporary data for 069dbda53fbb065f68f88519d6f94bc4 2024-12-07T15:16:05,514 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-07T15:16:05,515 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(1093): writing seq id for 069dbda53fbb065f68f88519d6f94bc4 2024-12-07T15:16:05,515 INFO [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(1114): Opened 069dbda53fbb065f68f88519d6f94bc4; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70341258, jitterRate=0.048166424036026}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-07T15:16:05,515 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 069dbda53fbb065f68f88519d6f94bc4 2024-12-07T15:16:05,516 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(1006): Region open journal for 069dbda53fbb065f68f88519d6f94bc4: Running coprocessor pre-open hook at 1733584565507Writing region info on filesystem at 1733584565507Initializing all the Stores at 1733584565507Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733584565507Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733584565508 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733584565508Cleaning up temporary data from old regions at 1733584565514 (+6 ms)Running coprocessor post-open hooks at 1733584565516 (+2 ms)Region opened successfully at 1733584565516 2024-12-07T15:16:05,517 INFO [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., pid=157, masterSystemTime=1733584565504 2024-12-07T15:16:05,518 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=155 updating hbase:meta row=069dbda53fbb065f68f88519d6f94bc4, regionState=OPEN, openSeqNum=5, regionLocation=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:05,519 DEBUG [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:05,519 INFO [RS_OPEN_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:05,520 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=157, ppid=155, state=RUNNABLE, hasLock=false; OpenRegionProcedure 069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 because future has completed 2024-12-07T15:16:05,521 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=155 2024-12-07T15:16:05,522 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=155, state=SUCCESS, hasLock=false; OpenRegionProcedure 069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 in 168 msec 2024-12-07T15:16:05,523 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-12-07T15:16:05,523 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=069dbda53fbb065f68f88519d6f94bc4, REOPEN/MOVE in 486 msec 2024-12-07T15:16:05,525 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=154, resume processing ppid=153 2024-12-07T15:16:05,525 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, ppid=153, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 489 msec 2024-12-07T15:16:05,526 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 903 msec 2024-12-07T15:16:05,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=153 2024-12-07T15:16:05,527 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-07T15:16:05,527 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11971b93, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:16:05,527 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6797ebe9c52,36915,-1 for getting cluster id 2024-12-07T15:16:05,528 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T15:16:05,528 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f8956274-ed33-4e16-82eb-d5d5911ec893' 2024-12-07T15:16:05,528 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T15:16:05,529 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f8956274-ed33-4e16-82eb-d5d5911ec893" 2024-12-07T15:16:05,529 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@648d89, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:16:05,529 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6797ebe9c52,36915,-1] 2024-12-07T15:16:05,529 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T15:16:05,529 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:16:05,530 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53368, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T15:16:05,530 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@134574c0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:16:05,531 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24560e66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:16:05,531 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6797ebe9c52,36915,-1 for getting cluster id 2024-12-07T15:16:05,531 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T15:16:05,531 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f8956274-ed33-4e16-82eb-d5d5911ec893' 2024-12-07T15:16:05,531 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T15:16:05,531 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f8956274-ed33-4e16-82eb-d5d5911ec893" 2024-12-07T15:16:05,531 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b543977, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:16:05,531 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6797ebe9c52,36915,-1] 2024-12-07T15:16:05,532 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T15:16:05,532 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:16:05,532 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53392, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T15:16:05,533 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5111a855, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:16:05,533 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e83be8c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:16:05,533 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6797ebe9c52,36915,-1 for getting cluster id 2024-12-07T15:16:05,533 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T15:16:05,534 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f8956274-ed33-4e16-82eb-d5d5911ec893' 2024-12-07T15:16:05,534 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T15:16:05,534 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f8956274-ed33-4e16-82eb-d5d5911ec893" 2024-12-07T15:16:05,534 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58a33b31, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:16:05,534 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6797ebe9c52,36915,-1] 2024-12-07T15:16:05,534 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T15:16:05,534 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:16:05,535 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53404, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T15:16:05,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74ca5523, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:16:05,536 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49c8d9ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:16:05,536 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6797ebe9c52,36915,-1 for getting cluster id 2024-12-07T15:16:05,536 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T15:16:05,536 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f8956274-ed33-4e16-82eb-d5d5911ec893' 2024-12-07T15:16:05,537 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T15:16:05,537 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f8956274-ed33-4e16-82eb-d5d5911ec893" 2024-12-07T15:16:05,537 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b4c38dd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:16:05,537 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6797ebe9c52,36915,-1] 2024-12-07T15:16:05,537 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T15:16:05,537 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:16:05,538 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53428, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T15:16:05,538 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59b72b62, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:16:05,538 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37f0cd5b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:16:05,538 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6797ebe9c52,36915,-1 for getting cluster id 2024-12-07T15:16:05,538 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T15:16:05,539 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f8956274-ed33-4e16-82eb-d5d5911ec893' 2024-12-07T15:16:05,539 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T15:16:05,539 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f8956274-ed33-4e16-82eb-d5d5911ec893" 2024-12-07T15:16:05,539 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36531420, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:16:05,539 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6797ebe9c52,36915,-1] 2024-12-07T15:16:05,539 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T15:16:05,539 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:16:05,540 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53446, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T15:16:05,541 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@217364b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:16:05,541 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e0b6ba1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:16:05,541 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6797ebe9c52,36915,-1 for getting cluster id 2024-12-07T15:16:05,541 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T15:16:05,542 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f8956274-ed33-4e16-82eb-d5d5911ec893' 2024-12-07T15:16:05,542 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T15:16:05,542 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f8956274-ed33-4e16-82eb-d5d5911ec893" 2024-12-07T15:16:05,542 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f2d52cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:16:05,542 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6797ebe9c52,36915,-1] 2024-12-07T15:16:05,542 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T15:16:05,542 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:16:05,543 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53476, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T15:16:05,543 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@427d3ae9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:16:05,544 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19bc8537, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:16:05,544 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6797ebe9c52,36915,-1 for getting cluster id 2024-12-07T15:16:05,544 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T15:16:05,544 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f8956274-ed33-4e16-82eb-d5d5911ec893' 2024-12-07T15:16:05,544 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T15:16:05,545 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f8956274-ed33-4e16-82eb-d5d5911ec893" 2024-12-07T15:16:05,545 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65e94f29, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:16:05,545 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6797ebe9c52,36915,-1] 2024-12-07T15:16:05,545 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T15:16:05,545 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:16:05,545 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53486, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T15:16:05,546 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30613c31, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:16:05,547 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7595d14b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:16:05,547 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6797ebe9c52,36915,-1 for getting cluster id 2024-12-07T15:16:05,547 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T15:16:05,547 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f8956274-ed33-4e16-82eb-d5d5911ec893' 2024-12-07T15:16:05,547 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T15:16:05,547 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f8956274-ed33-4e16-82eb-d5d5911ec893" 2024-12-07T15:16:05,548 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@689b6ff2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:16:05,548 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6797ebe9c52,36915,-1] 2024-12-07T15:16:05,548 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T15:16:05,548 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:16:05,548 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53502, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T15:16:05,549 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29e25e28, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:16:05,549 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bf79e45, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:16:05,549 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6797ebe9c52,36915,-1 for getting cluster id 2024-12-07T15:16:05,549 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T15:16:05,550 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f8956274-ed33-4e16-82eb-d5d5911ec893' 2024-12-07T15:16:05,550 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T15:16:05,550 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f8956274-ed33-4e16-82eb-d5d5911ec893" 2024-12-07T15:16:05,550 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ad6bfef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:16:05,550 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6797ebe9c52,36915,-1] 2024-12-07T15:16:05,550 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T15:16:05,551 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:16:05,551 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53518, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T15:16:05,552 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26448eee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:16:05,552 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23d0e9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:16:05,552 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d6797ebe9c52,36915,-1 for getting cluster id 2024-12-07T15:16:05,552 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T15:16:05,553 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f8956274-ed33-4e16-82eb-d5d5911ec893' 2024-12-07T15:16:05,553 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T15:16:05,553 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f8956274-ed33-4e16-82eb-d5d5911ec893" 2024-12-07T15:16:05,553 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23ee4f47, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:16:05,553 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d6797ebe9c52,36915,-1] 2024-12-07T15:16:05,553 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T15:16:05,553 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:16:05,554 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53526, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T15:16:05,554 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c3103d4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T15:16:05,555 DEBUG [Thread-2400 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T15:16:05,555 DEBUG [Thread-2404 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T15:16:05,555 DEBUG [Thread-2406 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T15:16:05,555 DEBUG [Thread-2398 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T15:16:05,555 DEBUG [Thread-2391 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T15:16:05,555 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T15:16:05,555 DEBUG [Thread-2389 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T15:16:05,556 DEBUG [Thread-2395 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T15:16:05,556 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6797ebe9c52,46009,1733584418401, seqNum=-1] 2024-12-07T15:16:05,556 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6797ebe9c52,46009,1733584418401, seqNum=-1] 2024-12-07T15:16:05,556 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T15:16:05,556 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T15:16:05,556 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6797ebe9c52,46009,1733584418401, seqNum=-1] 2024-12-07T15:16:05,556 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T15:16:05,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-12-07T15:16:05,556 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6797ebe9c52,46009,1733584418401, seqNum=-1] 2024-12-07T15:16:05,556 DEBUG [Thread-2387 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T15:16:05,556 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6797ebe9c52,46009,1733584418401, seqNum=-1] 2024-12-07T15:16:05,556 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T15:16:05,557 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T15:16:05,557 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44628, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T15:16:05,557 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6797ebe9c52,46009,1733584418401, seqNum=-1] 2024-12-07T15:16:05,557 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T15:16:05,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-07T15:16:05,557 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44640, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T15:16:05,557 DEBUG [Thread-2406 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6797ebe9c52,46009,1733584418401, seqNum=-1] 2024-12-07T15:16:05,557 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44646, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T15:16:05,558 DEBUG [Thread-2406 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T15:16:05,558 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6797ebe9c52,46009,1733584418401, seqNum=-1] 2024-12-07T15:16:05,558 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44650, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T15:16:05,558 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T15:16:05,558 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44654, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T15:16:05,558 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T15:16:05,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5] 2024-12-07T15:16:05,559 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44662, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T15:16:05,559 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44672, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T15:16:05,559 DEBUG [Thread-2402 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T15:16:05,559 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44682, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T15:16:05,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5] 2024-12-07T15:16:05,559 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T15:16:05,559 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T15:16:05,560 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6797ebe9c52,46009,1733584418401, seqNum=-1] 2024-12-07T15:16:05,560 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5] 2024-12-07T15:16:05,560 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5] 2024-12-07T15:16:05,560 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T15:16:05,560 DEBUG [Thread-2393 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T15:16:05,561 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5] 2024-12-07T15:16:05,561 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5] 2024-12-07T15:16:05,561 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44688, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T15:16:05,562 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5] 2024-12-07T15:16:05,562 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d6797ebe9c52,46009,1733584418401, seqNum=-1] 2024-12-07T15:16:05,562 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T15:16:05,563 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5] 2024-12-07T15:16:05,563 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5] 2024-12-07T15:16:05,565 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44694, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T15:16:05,567 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5] 2024-12-07T15:16:05,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on 069dbda53fbb065f68f88519d6f94bc4 2024-12-07T15:16:05,568 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 069dbda53fbb065f68f88519d6f94bc4 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-07T15:16:05,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 069dbda53fbb065f68f88519d6f94bc4, store=A 2024-12-07T15:16:05,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:16:05,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 069dbda53fbb065f68f88519d6f94bc4, store=B 2024-12-07T15:16:05,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:16:05,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 069dbda53fbb065f68f88519d6f94bc4, store=C 2024-12-07T15:16:05,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:16:05,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:05,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44694 deadline: 1733584625583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:05,590 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:05,590 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:05,590 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:05,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:05,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44662 deadline: 1733584625584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:05,591 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:05,591 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:05,591 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:05,591 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733584565581, max timestamp=1733584565588Num uniques 859; 2024-12-07T15:16:05,591 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-07T15:16:05,591 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733584565577, max timestamp=1733584565588Num uniques 859; 2024-12-07T15:16:05,592 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-07T15:16:05,592 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733584565581, max timestamp=1733584565588Num uniques 859; 2024-12-07T15:16:05,592 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-07T15:16:05,595 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120753121cf77dde43ecb431f55203845870_069dbda53fbb065f68f88519d6f94bc4 is 50, key is test_row_0/A:col10/1733584565567/Put/seqid=0 2024-12-07T15:16:05,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742361_1537 (size=12154) 2024-12-07T15:16:05,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:05,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44654 deadline: 1733584625649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:05,656 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:05,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:05,656 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:05,656 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:05,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44682 deadline: 1733584625649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:05,656 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:05,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:05,656 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:05,656 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:05,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44650 deadline: 1733584625649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:05,657 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:05,657 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:05,657 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:05,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-07T15:16:05,696 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:05,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44694 deadline: 1733584625693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:05,696 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:05,696 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:05,696 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:05,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44662 deadline: 1733584625693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:05,696 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:05,696 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:05,697 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:05,697 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:05,714 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-07T15:16:05,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:05,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. as already flushing 2024-12-07T15:16:05,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:05,714 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:16:05,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:16:05,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:16:05,766 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:05,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44654 deadline: 1733584625763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:05,767 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:05,767 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:05,767 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:05,767 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:05,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44682 deadline: 1733584625763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:05,767 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:05,767 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:05,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44650 deadline: 1733584625763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:05,767 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:05,767 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:05,767 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:05,768 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:05,768 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:05,867 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-07T15:16:05,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:05,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. as already flushing 2024-12-07T15:16:05,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:05,868 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:16:05,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:16:05,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:16:05,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-07T15:16:05,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:05,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44694 deadline: 1733584625903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:05,907 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:05,907 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:05,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:05,907 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:05,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44662 deadline: 1733584625903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:05,908 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:05,908 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:05,908 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:05,976 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:05,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44654 deadline: 1733584625973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:05,976 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:05,976 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:05,976 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:05,977 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:05,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44682 deadline: 1733584625973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:05,977 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:05,977 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:05,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44650 deadline: 1733584625973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:05,977 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:05,977 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:05,978 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:05,978 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:05,978 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:06,004 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:16:06,007 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120753121cf77dde43ecb431f55203845870_069dbda53fbb065f68f88519d6f94bc4 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120753121cf77dde43ecb431f55203845870_069dbda53fbb065f68f88519d6f94bc4 2024-12-07T15:16:06,008 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/A/8a7da40c81a14b9caf61636ea65f1e3c, store: [table=TestAcidGuarantees family=A region=069dbda53fbb065f68f88519d6f94bc4] 2024-12-07T15:16:06,008 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/A/8a7da40c81a14b9caf61636ea65f1e3c is 175, key is test_row_0/A:col10/1733584565567/Put/seqid=0 2024-12-07T15:16:06,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742362_1538 (size=30955) 2024-12-07T15:16:06,020 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-07T15:16:06,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:06,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. as already flushing 2024-12-07T15:16:06,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:06,020 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:16:06,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:16:06,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:16:06,173 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-07T15:16:06,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:06,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. as already flushing 2024-12-07T15:16:06,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:06,173 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:16:06,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:16:06,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:16:06,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-07T15:16:06,216 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:06,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44694 deadline: 1733584626213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:06,216 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:06,216 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:06,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44662 deadline: 1733584626213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:06,216 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:06,216 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:06,216 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:06,216 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:06,216 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:06,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:06,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44654 deadline: 1733584626283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:06,285 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:06,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:06,285 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:06,285 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:06,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44682 deadline: 1733584626283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:06,285 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:06,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:06,285 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:06,285 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:06,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44650 deadline: 1733584626283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:06,285 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:06,285 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:06,285 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:06,326 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-07T15:16:06,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:06,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. as already flushing 2024-12-07T15:16:06,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:06,326 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:16:06,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:16:06,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:16:06,413 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/A/8a7da40c81a14b9caf61636ea65f1e3c 2024-12-07T15:16:06,433 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/B/e5d6283820c646069ecadd8849d6a487 is 50, key is test_row_0/B:col10/1733584565567/Put/seqid=0 2024-12-07T15:16:06,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742363_1539 (size=12001) 2024-12-07T15:16:06,478 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-07T15:16:06,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:06,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. as already flushing 2024-12-07T15:16:06,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:06,479 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:16:06,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:16:06,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:16:06,637 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-07T15:16:06,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:06,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. as already flushing 2024-12-07T15:16:06,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:06,638 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:16:06,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:16:06,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:16:06,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-07T15:16:06,724 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:06,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44694 deadline: 1733584626723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:06,724 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:06,725 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:06,725 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:06,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44662 deadline: 1733584626723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:06,725 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:06,725 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:06,725 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:06,725 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:06,776 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-07T15:16:06,790 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-07T15:16:06,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:06,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. as already flushing 2024-12-07T15:16:06,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:06,790 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:16:06,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:16:06,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:16:06,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:06,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44654 deadline: 1733584626793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:06,797 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:06,797 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:06,797 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:06,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:06,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44682 deadline: 1733584626794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:06,798 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:06,798 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:06,798 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:06,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:06,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44650 deadline: 1733584626794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:06,798 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:06,798 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:06,798 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:06,836 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/B/e5d6283820c646069ecadd8849d6a487 2024-12-07T15:16:06,857 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/C/7b7e83ef764645b59235d8f94087bc56 is 50, key is test_row_0/C:col10/1733584565567/Put/seqid=0 2024-12-07T15:16:06,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742364_1540 (size=12001) 2024-12-07T15:16:06,943 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-07T15:16:06,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:06,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. as already flushing 2024-12-07T15:16:06,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:06,944 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:16:06,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:16:06,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:16:07,096 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-07T15:16:07,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:07,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. as already flushing 2024-12-07T15:16:07,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:07,096 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:16:07,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:16:07,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:16:07,249 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-07T15:16:07,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:07,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. as already flushing 2024-12-07T15:16:07,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:07,250 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:16:07,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:16:07,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:16:07,263 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/C/7b7e83ef764645b59235d8f94087bc56 2024-12-07T15:16:07,268 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/A/8a7da40c81a14b9caf61636ea65f1e3c as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/8a7da40c81a14b9caf61636ea65f1e3c 2024-12-07T15:16:07,272 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/8a7da40c81a14b9caf61636ea65f1e3c, entries=150, sequenceid=15, filesize=30.2 K 2024-12-07T15:16:07,274 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/B/e5d6283820c646069ecadd8849d6a487 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/e5d6283820c646069ecadd8849d6a487 2024-12-07T15:16:07,282 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/e5d6283820c646069ecadd8849d6a487, entries=150, sequenceid=15, filesize=11.7 K 2024-12-07T15:16:07,283 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/C/7b7e83ef764645b59235d8f94087bc56 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/7b7e83ef764645b59235d8f94087bc56 2024-12-07T15:16:07,288 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/7b7e83ef764645b59235d8f94087bc56, entries=150, sequenceid=15, filesize=11.7 K 2024-12-07T15:16:07,289 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=136.42 KB/139690 for 069dbda53fbb065f68f88519d6f94bc4 in 1721ms, sequenceid=15, compaction requested=false 2024-12-07T15:16:07,289 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 069dbda53fbb065f68f88519d6f94bc4: 2024-12-07T15:16:07,405 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-07T15:16:07,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:07,405 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing 069dbda53fbb065f68f88519d6f94bc4 3/3 column families, dataSize=136.42 KB heapSize=385.38 KB 2024-12-07T15:16:07,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 069dbda53fbb065f68f88519d6f94bc4, store=A 2024-12-07T15:16:07,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T15:16:07,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 069dbda53fbb065f68f88519d6f94bc4, store=B 2024-12-07T15:16:07,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T15:16:07,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 069dbda53fbb065f68f88519d6f94bc4, store=C 2024-12-07T15:16:07,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T15:16:07,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207bf89dec20d7f4626ae0290b83100165d_069dbda53fbb065f68f88519d6f94bc4 is 50, key is test_row_0/A:col10/1733584565626/Put/seqid=0 2024-12-07T15:16:07,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742365_1541 (size=12154) 2024-12-07T15:16:07,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:16:07,466 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207bf89dec20d7f4626ae0290b83100165d_069dbda53fbb065f68f88519d6f94bc4 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207bf89dec20d7f4626ae0290b83100165d_069dbda53fbb065f68f88519d6f94bc4 2024-12-07T15:16:07,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/A/807f4cbdc9724a3cbf0669d9ec00b6b0, store: [table=TestAcidGuarantees family=A region=069dbda53fbb065f68f88519d6f94bc4] 2024-12-07T15:16:07,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/A/807f4cbdc9724a3cbf0669d9ec00b6b0 is 175, key is test_row_0/A:col10/1733584565626/Put/seqid=0 2024-12-07T15:16:07,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742366_1542 (size=30955) 2024-12-07T15:16:07,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-07T15:16:07,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on 069dbda53fbb065f68f88519d6f94bc4 2024-12-07T15:16:07,746 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. as already flushing 2024-12-07T15:16:07,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:07,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44662 deadline: 1733584627758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:07,763 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:07,763 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:07,763 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:07,765 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-07T15:16:07,767 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:07,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44694 deadline: 1733584627762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:07,767 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:07,767 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:07,767 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:07,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:07,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44654 deadline: 1733584627813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:07,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:07,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44682 deadline: 1733584627813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:07,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:07,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44650 deadline: 1733584627813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:07,815 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to d6797ebe9c52,46009,1733584418401 for 'test_row_0' in 069dbda53fbb065f68f88519d6f94bc4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-07T15:16:07,816 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:07,816 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:07,816 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:07,815 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to d6797ebe9c52,46009,1733584418401 for 'test_row_2' in 069dbda53fbb065f68f88519d6f94bc4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-07T15:16:07,816 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:07,816 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:07,816 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to d6797ebe9c52,46009,1733584418401 for 'test_row_1' in 069dbda53fbb065f68f88519d6f94bc4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-07T15:16:07,816 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:07,816 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:07,816 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:07,816 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:07,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:07,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44662 deadline: 1733584627873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:07,876 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:07,876 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:07,876 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:07,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44694 deadline: 1733584627873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:07,876 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:07,876 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:07,876 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:07,876 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:07,889 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/A/807f4cbdc9724a3cbf0669d9ec00b6b0 2024-12-07T15:16:07,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/B/ff9fab01304244688f1d8ee13883a5de is 50, key is test_row_0/B:col10/1733584565626/Put/seqid=0 2024-12-07T15:16:07,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742367_1543 (size=12001) 2024-12-07T15:16:08,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:08,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44662 deadline: 1733584628083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:08,086 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:08,086 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:08,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44694 deadline: 1733584628083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:08,086 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:08,086 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:08,086 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:08,086 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:08,086 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:08,300 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/B/ff9fab01304244688f1d8ee13883a5de 2024-12-07T15:16:08,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/C/fc8c540115a04e5680fcce693e890ac9 is 50, key is test_row_0/C:col10/1733584565626/Put/seqid=0 2024-12-07T15:16:08,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742368_1544 (size=12001) 2024-12-07T15:16:08,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:08,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44662 deadline: 1733584628393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:08,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:08,394 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:08,395 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:08,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44694 deadline: 1733584628393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:08,395 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:08,395 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:08,395 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:08,395 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:08,711 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/C/fc8c540115a04e5680fcce693e890ac9 2024-12-07T15:16:08,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/A/807f4cbdc9724a3cbf0669d9ec00b6b0 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/807f4cbdc9724a3cbf0669d9ec00b6b0 2024-12-07T15:16:08,717 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/807f4cbdc9724a3cbf0669d9ec00b6b0, entries=150, sequenceid=51, filesize=30.2 K 2024-12-07T15:16:08,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/B/ff9fab01304244688f1d8ee13883a5de as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/ff9fab01304244688f1d8ee13883a5de 2024-12-07T15:16:08,721 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/ff9fab01304244688f1d8ee13883a5de, entries=150, sequenceid=51, filesize=11.7 K 2024-12-07T15:16:08,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/C/fc8c540115a04e5680fcce693e890ac9 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/fc8c540115a04e5680fcce693e890ac9 2024-12-07T15:16:08,724 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/fc8c540115a04e5680fcce693e890ac9, entries=150, sequenceid=51, filesize=11.7 K 2024-12-07T15:16:08,725 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~136.42 KB/139690, heapSize ~385.33 KB/394576, currentSize=46.96 KB/48090 for 069dbda53fbb065f68f88519d6f94bc4 in 1320ms, sequenceid=51, compaction requested=false 2024-12-07T15:16:08,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for 069dbda53fbb065f68f88519d6f94bc4: 2024-12-07T15:16:08,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:08,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-12-07T15:16:08,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-12-07T15:16:08,728 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-12-07T15:16:08,728 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1670 sec 2024-12-07T15:16:08,730 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 3.1740 sec 2024-12-07T15:16:08,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on 069dbda53fbb065f68f88519d6f94bc4 2024-12-07T15:16:08,905 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 069dbda53fbb065f68f88519d6f94bc4 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-07T15:16:08,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 069dbda53fbb065f68f88519d6f94bc4, store=A 2024-12-07T15:16:08,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:16:08,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 069dbda53fbb065f68f88519d6f94bc4, store=B 2024-12-07T15:16:08,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:16:08,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 069dbda53fbb065f68f88519d6f94bc4, store=C 2024-12-07T15:16:08,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:16:08,911 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412075d9de84e3ff6491dbe135d9db1d0184c_069dbda53fbb065f68f88519d6f94bc4 is 50, key is test_row_0/A:col10/1733584567760/Put/seqid=0 2024-12-07T15:16:08,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742369_1545 (size=12154) 2024-12-07T15:16:08,969 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733584568939, max timestamp=1733584568952Num uniques 859; 2024-12-07T15:16:08,969 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-07T15:16:08,969 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733584568939, max timestamp=1733584568952Num uniques 859; 2024-12-07T15:16:08,969 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733584568939, max timestamp=1733584568952Num uniques 859; 2024-12-07T15:16:08,969 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-07T15:16:08,969 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/d6797ebe9c52:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-07T15:16:09,016 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:09,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44694 deadline: 1733584629010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:09,016 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:09,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44662 deadline: 1733584629011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:09,016 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:09,016 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:09,016 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:09,017 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:09,017 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:09,017 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:09,127 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:09,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44694 deadline: 1733584629123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:09,127 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:09,127 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:09,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44662 deadline: 1733584629123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:09,127 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:09,127 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:09,127 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:09,127 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:09,127 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:09,315 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:16:09,319 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412075d9de84e3ff6491dbe135d9db1d0184c_069dbda53fbb065f68f88519d6f94bc4 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412075d9de84e3ff6491dbe135d9db1d0184c_069dbda53fbb065f68f88519d6f94bc4 2024-12-07T15:16:09,319 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/A/d8819e606cf14e558a68af6cc2cc5299, store: [table=TestAcidGuarantees family=A region=069dbda53fbb065f68f88519d6f94bc4] 2024-12-07T15:16:09,320 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/A/d8819e606cf14e558a68af6cc2cc5299 is 175, key is test_row_0/A:col10/1733584567760/Put/seqid=0 2024-12-07T15:16:09,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742370_1546 (size=30955) 2024-12-07T15:16:09,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:09,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44694 deadline: 1733584629333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:09,338 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:09,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:09,338 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:09,338 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:09,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44662 deadline: 1733584629333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:09,338 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:09,338 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:09,338 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:09,645 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:09,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44694 deadline: 1733584629643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:09,645 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:09,645 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:09,645 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:09,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44662 deadline: 1733584629643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:09,645 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:09,645 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:09,645 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:09,646 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:09,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-07T15:16:09,723 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T15:16:09,723 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T15:16:09,724 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=62, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/A/d8819e606cf14e558a68af6cc2cc5299 2024-12-07T15:16:09,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-12-07T15:16:09,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-07T15:16:09,725 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T15:16:09,726 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T15:16:09,726 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T15:16:09,731 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/B/579e1cd0111f47c98d488193543a2cb0 is 50, key is test_row_0/B:col10/1733584567760/Put/seqid=0 2024-12-07T15:16:09,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742371_1547 (size=12001) 2024-12-07T15:16:09,827 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:09,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44682 deadline: 1733584629823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:09,828 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to d6797ebe9c52,46009,1733584418401 for 'test_row_2' in 069dbda53fbb065f68f88519d6f94bc4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-07T15:16:09,829 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:09,829 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:09,829 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:09,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-07T15:16:09,834 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:09,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44654 deadline: 1733584629833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:09,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:09,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44650 deadline: 1733584629833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:09,835 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to d6797ebe9c52,46009,1733584418401 for 'test_row_0' in 069dbda53fbb065f68f88519d6f94bc4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-07T15:16:09,835 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:09,835 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:09,835 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to d6797ebe9c52,46009,1733584418401 for 'test_row_1' in 069dbda53fbb065f68f88519d6f94bc4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-07T15:16:09,835 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:09,835 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:09,836 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:09,836 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:09,878 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-07T15:16:09,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:09,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. as already flushing 2024-12-07T15:16:09,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:09,879 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:16:09,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:16:09,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:16:10,031 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-07T15:16:10,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:10,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. as already flushing 2024-12-07T15:16:10,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:10,032 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:16:10,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:16:10,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:16:10,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-07T15:16:10,135 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/B/579e1cd0111f47c98d488193543a2cb0 2024-12-07T15:16:10,142 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/C/3c9d1ff9ee794047b33ae2fd9df73b05 is 50, key is test_row_0/C:col10/1733584567760/Put/seqid=0 2024-12-07T15:16:10,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742372_1548 (size=12001) 2024-12-07T15:16:10,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:10,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44694 deadline: 1733584630153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:10,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:10,157 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:10,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44662 deadline: 1733584630154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:10,157 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:10,158 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:10,158 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:10,158 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:10,158 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:10,184 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-07T15:16:10,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:10,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. as already flushing 2024-12-07T15:16:10,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:10,185 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:16:10,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:16:10,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:16:10,337 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-07T15:16:10,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:10,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. as already flushing 2024-12-07T15:16:10,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:10,338 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:16:10,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:16:10,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:16:10,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-07T15:16:10,489 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-07T15:16:10,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:10,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. as already flushing 2024-12-07T15:16:10,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:10,490 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:16:10,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:16:10,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:16:10,546 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/C/3c9d1ff9ee794047b33ae2fd9df73b05 2024-12-07T15:16:10,550 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/A/d8819e606cf14e558a68af6cc2cc5299 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/d8819e606cf14e558a68af6cc2cc5299 2024-12-07T15:16:10,553 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/d8819e606cf14e558a68af6cc2cc5299, entries=150, sequenceid=62, filesize=30.2 K 2024-12-07T15:16:10,554 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/B/579e1cd0111f47c98d488193543a2cb0 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/579e1cd0111f47c98d488193543a2cb0 2024-12-07T15:16:10,557 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/579e1cd0111f47c98d488193543a2cb0, entries=150, sequenceid=62, filesize=11.7 K 2024-12-07T15:16:10,558 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/C/3c9d1ff9ee794047b33ae2fd9df73b05 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/3c9d1ff9ee794047b33ae2fd9df73b05 2024-12-07T15:16:10,561 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/3c9d1ff9ee794047b33ae2fd9df73b05, entries=150, sequenceid=62, filesize=11.7 K 2024-12-07T15:16:10,562 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 069dbda53fbb065f68f88519d6f94bc4 in 1657ms, sequenceid=62, compaction requested=true 2024-12-07T15:16:10,562 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 069dbda53fbb065f68f88519d6f94bc4: 2024-12-07T15:16:10,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 069dbda53fbb065f68f88519d6f94bc4:A, priority=-2147483648, current under compaction store size is 1 2024-12-07T15:16:10,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:16:10,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 069dbda53fbb065f68f88519d6f94bc4:B, priority=-2147483648, current under compaction store size is 2 2024-12-07T15:16:10,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:16:10,562 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:16:10,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 069dbda53fbb065f68f88519d6f94bc4:C, priority=-2147483648, current under compaction store size is 3 2024-12-07T15:16:10,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T15:16:10,562 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:16:10,565 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:16:10,565 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:16:10,565 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1541): 069dbda53fbb065f68f88519d6f94bc4/B is initiating minor compaction (all files) 2024-12-07T15:16:10,565 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 069dbda53fbb065f68f88519d6f94bc4/B in TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:10,565 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/e5d6283820c646069ecadd8849d6a487, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/ff9fab01304244688f1d8ee13883a5de, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/579e1cd0111f47c98d488193543a2cb0] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp, totalSize=35.2 K 2024-12-07T15:16:10,565 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1541): 069dbda53fbb065f68f88519d6f94bc4/A is initiating minor compaction (all files) 2024-12-07T15:16:10,565 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 069dbda53fbb065f68f88519d6f94bc4/A in TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:10,565 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/8a7da40c81a14b9caf61636ea65f1e3c, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/807f4cbdc9724a3cbf0669d9ec00b6b0, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/d8819e606cf14e558a68af6cc2cc5299] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp, totalSize=90.7 K 2024-12-07T15:16:10,565 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting e5d6283820c646069ecadd8849d6a487, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733584565564 2024-12-07T15:16:10,565 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:10,565 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. files: [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/8a7da40c81a14b9caf61636ea65f1e3c, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/807f4cbdc9724a3cbf0669d9ec00b6b0, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/d8819e606cf14e558a68af6cc2cc5299] 2024-12-07T15:16:10,566 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting ff9fab01304244688f1d8ee13883a5de, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733584565613 2024-12-07T15:16:10,566 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a7da40c81a14b9caf61636ea65f1e3c, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733584565564 2024-12-07T15:16:10,566 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 807f4cbdc9724a3cbf0669d9ec00b6b0, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733584565613 2024-12-07T15:16:10,566 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 579e1cd0111f47c98d488193543a2cb0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733584567757 2024-12-07T15:16:10,566 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8819e606cf14e558a68af6cc2cc5299, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733584567757 2024-12-07T15:16:10,571 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=069dbda53fbb065f68f88519d6f94bc4] 2024-12-07T15:16:10,573 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 069dbda53fbb065f68f88519d6f94bc4#B#compaction#470 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:16:10,574 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/B/2a339d6a654e4b86940f554eae331e4e is 50, key is test_row_0/B:col10/1733584567760/Put/seqid=0 2024-12-07T15:16:10,575 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241207663f6501405b48aabe21c63a967c4b04_069dbda53fbb065f68f88519d6f94bc4 store=[table=TestAcidGuarantees family=A region=069dbda53fbb065f68f88519d6f94bc4] 2024-12-07T15:16:10,577 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241207663f6501405b48aabe21c63a967c4b04_069dbda53fbb065f68f88519d6f94bc4, store=[table=TestAcidGuarantees family=A region=069dbda53fbb065f68f88519d6f94bc4] 2024-12-07T15:16:10,578 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207663f6501405b48aabe21c63a967c4b04_069dbda53fbb065f68f88519d6f94bc4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=069dbda53fbb065f68f88519d6f94bc4] 2024-12-07T15:16:10,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742373_1549 (size=12104) 2024-12-07T15:16:10,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742374_1550 (size=4469) 2024-12-07T15:16:10,642 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-07T15:16:10,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:10,642 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing 069dbda53fbb065f68f88519d6f94bc4 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-12-07T15:16:10,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 069dbda53fbb065f68f88519d6f94bc4, store=A 2024-12-07T15:16:10,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T15:16:10,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 069dbda53fbb065f68f88519d6f94bc4, store=B 2024-12-07T15:16:10,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T15:16:10,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 069dbda53fbb065f68f88519d6f94bc4, store=C 2024-12-07T15:16:10,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T15:16:10,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412074fce63f886e745b6ada9b0967cd8585e_069dbda53fbb065f68f88519d6f94bc4 is 50, key is test_row_0/A:col10/1733584569009/Put/seqid=0 2024-12-07T15:16:10,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742375_1551 (size=12154) 2024-12-07T15:16:10,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-07T15:16:10,986 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/B/2a339d6a654e4b86940f554eae331e4e as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/2a339d6a654e4b86940f554eae331e4e 2024-12-07T15:16:10,992 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 069dbda53fbb065f68f88519d6f94bc4/B of 069dbda53fbb065f68f88519d6f94bc4 into 2a339d6a654e4b86940f554eae331e4e(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:16:10,992 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 069dbda53fbb065f68f88519d6f94bc4: 2024-12-07T15:16:10,992 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., storeName=069dbda53fbb065f68f88519d6f94bc4/B, priority=13, startTime=1733584570562; duration=0sec 2024-12-07T15:16:10,993 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T15:16:10,993 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 069dbda53fbb065f68f88519d6f94bc4:B 2024-12-07T15:16:10,993 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:16:10,993 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 069dbda53fbb065f68f88519d6f94bc4#A#compaction#469 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:16:10,993 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:16:10,994 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1541): 069dbda53fbb065f68f88519d6f94bc4/C is initiating minor compaction (all files) 2024-12-07T15:16:10,994 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 069dbda53fbb065f68f88519d6f94bc4/C in TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:10,994 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/7b7e83ef764645b59235d8f94087bc56, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/fc8c540115a04e5680fcce693e890ac9, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/3c9d1ff9ee794047b33ae2fd9df73b05] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp, totalSize=35.2 K 2024-12-07T15:16:10,994 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/A/aa2b61a68ad344a5b9247a5ddffb8dd9 is 175, key is test_row_0/A:col10/1733584567760/Put/seqid=0 2024-12-07T15:16:10,994 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b7e83ef764645b59235d8f94087bc56, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733584565564 2024-12-07T15:16:10,995 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting fc8c540115a04e5680fcce693e890ac9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733584565613 2024-12-07T15:16:10,995 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c9d1ff9ee794047b33ae2fd9df73b05, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733584567757 2024-12-07T15:16:10,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742376_1552 (size=31058) 2024-12-07T15:16:11,001 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 069dbda53fbb065f68f88519d6f94bc4#C#compaction#472 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:16:11,002 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/C/bf759a03a63644ce8bbeace14a4c8b58 is 50, key is test_row_0/C:col10/1733584567760/Put/seqid=0 2024-12-07T15:16:11,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742377_1553 (size=12104) 2024-12-07T15:16:11,011 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/C/bf759a03a63644ce8bbeace14a4c8b58 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/bf759a03a63644ce8bbeace14a4c8b58 2024-12-07T15:16:11,015 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 069dbda53fbb065f68f88519d6f94bc4/C of 069dbda53fbb065f68f88519d6f94bc4 into bf759a03a63644ce8bbeace14a4c8b58(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:16:11,015 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 069dbda53fbb065f68f88519d6f94bc4: 2024-12-07T15:16:11,015 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., storeName=069dbda53fbb065f68f88519d6f94bc4/C, priority=13, startTime=1733584570562; duration=0sec 2024-12-07T15:16:11,015 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:16:11,015 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 069dbda53fbb065f68f88519d6f94bc4:C 2024-12-07T15:16:11,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:16:11,055 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412074fce63f886e745b6ada9b0967cd8585e_069dbda53fbb065f68f88519d6f94bc4 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412074fce63f886e745b6ada9b0967cd8585e_069dbda53fbb065f68f88519d6f94bc4 2024-12-07T15:16:11,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/A/df5054d67b1645de8116f9f6fa90796d, store: [table=TestAcidGuarantees family=A region=069dbda53fbb065f68f88519d6f94bc4] 2024-12-07T15:16:11,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/A/df5054d67b1645de8116f9f6fa90796d is 175, key is test_row_0/A:col10/1733584569009/Put/seqid=0 2024-12-07T15:16:11,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742378_1554 (size=30955) 2024-12-07T15:16:11,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on 069dbda53fbb065f68f88519d6f94bc4 2024-12-07T15:16:11,176 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. as already flushing 2024-12-07T15:16:11,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:11,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44662 deadline: 1733584631190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:11,195 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:11,195 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:11,195 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:11,197 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:11,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44694 deadline: 1733584631192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:11,198 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:11,198 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:11,198 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:11,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:11,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44662 deadline: 1733584631303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:11,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:11,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44694 deadline: 1733584631303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:11,305 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:11,305 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:11,305 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:11,306 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:11,306 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:11,306 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:11,401 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/A/aa2b61a68ad344a5b9247a5ddffb8dd9 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/aa2b61a68ad344a5b9247a5ddffb8dd9 2024-12-07T15:16:11,405 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 069dbda53fbb065f68f88519d6f94bc4/A of 069dbda53fbb065f68f88519d6f94bc4 into aa2b61a68ad344a5b9247a5ddffb8dd9(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:16:11,405 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 069dbda53fbb065f68f88519d6f94bc4: 2024-12-07T15:16:11,405 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., storeName=069dbda53fbb065f68f88519d6f94bc4/A, priority=13, startTime=1733584570562; duration=0sec 2024-12-07T15:16:11,405 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:16:11,405 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 069dbda53fbb065f68f88519d6f94bc4:A 2024-12-07T15:16:11,461 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=98, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/A/df5054d67b1645de8116f9f6fa90796d 2024-12-07T15:16:11,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/B/6147dbaaf78e4e03adaa546b103f1273 is 50, key is test_row_0/B:col10/1733584569009/Put/seqid=0 2024-12-07T15:16:11,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742379_1555 (size=12001) 2024-12-07T15:16:11,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:11,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44662 deadline: 1733584631513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:11,517 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:11,517 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:11,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44694 deadline: 1733584631514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:11,517 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:11,517 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:11,517 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:11,517 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:11,517 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:11,825 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:11,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44662 deadline: 1733584631823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:11,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:11,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44694 deadline: 1733584631823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:11,826 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:11,826 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:11,826 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:11,826 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:11,826 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:11,826 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:11,871 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/B/6147dbaaf78e4e03adaa546b103f1273 2024-12-07T15:16:11,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-07T15:16:11,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/C/3c99821889c24a41b4c0e1662092eec9 is 50, key is test_row_0/C:col10/1733584569009/Put/seqid=0 2024-12-07T15:16:11,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742380_1556 (size=12001) 2024-12-07T15:16:12,283 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/C/3c99821889c24a41b4c0e1662092eec9 2024-12-07T15:16:12,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/A/df5054d67b1645de8116f9f6fa90796d as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/df5054d67b1645de8116f9f6fa90796d 2024-12-07T15:16:12,290 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/df5054d67b1645de8116f9f6fa90796d, entries=150, sequenceid=98, filesize=30.2 K 2024-12-07T15:16:12,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/B/6147dbaaf78e4e03adaa546b103f1273 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/6147dbaaf78e4e03adaa546b103f1273 2024-12-07T15:16:12,293 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/6147dbaaf78e4e03adaa546b103f1273, entries=150, sequenceid=98, filesize=11.7 K 2024-12-07T15:16:12,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/C/3c99821889c24a41b4c0e1662092eec9 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/3c99821889c24a41b4c0e1662092eec9 2024-12-07T15:16:12,297 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/3c99821889c24a41b4c0e1662092eec9, entries=150, sequenceid=98, filesize=11.7 K 2024-12-07T15:16:12,298 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for 069dbda53fbb065f68f88519d6f94bc4 in 1656ms, sequenceid=98, compaction requested=false 2024-12-07T15:16:12,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for 069dbda53fbb065f68f88519d6f94bc4: 2024-12-07T15:16:12,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:12,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-12-07T15:16:12,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-12-07T15:16:12,301 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-12-07T15:16:12,301 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5730 sec 2024-12-07T15:16:12,303 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 2.5790 sec 2024-12-07T15:16:12,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on 069dbda53fbb065f68f88519d6f94bc4 2024-12-07T15:16:12,336 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 069dbda53fbb065f68f88519d6f94bc4 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-07T15:16:12,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 069dbda53fbb065f68f88519d6f94bc4, store=A 2024-12-07T15:16:12,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:16:12,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 069dbda53fbb065f68f88519d6f94bc4, store=B 2024-12-07T15:16:12,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:16:12,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 069dbda53fbb065f68f88519d6f94bc4, store=C 2024-12-07T15:16:12,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:16:12,350 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120794cba35461c34335bd8cf779ec5b3210_069dbda53fbb065f68f88519d6f94bc4 is 50, key is test_row_0/A:col10/1733584572335/Put/seqid=0 2024-12-07T15:16:12,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742381_1557 (size=14594) 2024-12-07T15:16:12,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:12,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44662 deadline: 1733584632375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:12,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:12,381 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:12,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44694 deadline: 1733584632377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:12,381 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:12,381 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:12,382 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:12,382 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:12,382 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:12,486 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:12,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44694 deadline: 1733584632483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:12,487 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:12,487 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:12,487 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:12,494 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:12,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44662 deadline: 1733584632493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:12,494 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:12,494 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:12,494 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:12,694 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:12,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44694 deadline: 1733584632693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:12,694 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:12,694 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:12,694 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:12,705 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:12,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44662 deadline: 1733584632703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:12,705 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:12,705 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:12,705 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:12,754 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:16:12,757 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120794cba35461c34335bd8cf779ec5b3210_069dbda53fbb065f68f88519d6f94bc4 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120794cba35461c34335bd8cf779ec5b3210_069dbda53fbb065f68f88519d6f94bc4 2024-12-07T15:16:12,758 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/A/9555352250874630bd2b7ab3f9919451, store: [table=TestAcidGuarantees family=A region=069dbda53fbb065f68f88519d6f94bc4] 2024-12-07T15:16:12,759 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/A/9555352250874630bd2b7ab3f9919451 is 175, key is test_row_0/A:col10/1733584572335/Put/seqid=0 2024-12-07T15:16:12,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742382_1558 (size=39549) 2024-12-07T15:16:13,007 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:13,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44694 deadline: 1733584633003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:13,008 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:13,008 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:13,008 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:13,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:13,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44662 deadline: 1733584633014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:13,017 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:13,017 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:13,018 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:13,163 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=114, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/A/9555352250874630bd2b7ab3f9919451 2024-12-07T15:16:13,169 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/B/8e11b48d6efe448e966c83661a5579d2 is 50, key is test_row_0/B:col10/1733584572335/Put/seqid=0 2024-12-07T15:16:13,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742383_1559 (size=12001) 2024-12-07T15:16:13,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:13,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44694 deadline: 1733584633513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:13,515 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:13,515 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:13,515 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:13,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:13,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44662 deadline: 1733584633523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:13,528 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:13,528 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:13,528 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:13,573 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/B/8e11b48d6efe448e966c83661a5579d2 2024-12-07T15:16:13,579 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/C/9dbe03f66c824444bed9465c040bb33c is 50, key is test_row_0/C:col10/1733584572335/Put/seqid=0 2024-12-07T15:16:13,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742384_1560 (size=12001) 2024-12-07T15:16:13,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:13,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44682 deadline: 1733584633843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:13,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:13,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44650 deadline: 1733584633843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:13,846 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to d6797ebe9c52,46009,1733584418401 for 'test_row_2' in 069dbda53fbb065f68f88519d6f94bc4 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-07T15:16:13,846 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to d6797ebe9c52,46009,1733584418401 for 'test_row_1' in 069dbda53fbb065f68f88519d6f94bc4 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-07T15:16:13,847 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:13,847 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:13,847 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:13,847 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:13,847 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:13,847 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:13,876 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:13,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44654 deadline: 1733584633873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:13,876 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to d6797ebe9c52,46009,1733584418401 for 'test_row_0' in 069dbda53fbb065f68f88519d6f94bc4 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8227 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-07T15:16:13,877 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:13,877 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:13,877 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:13,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-07T15:16:13,883 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T15:16:13,883 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T15:16:13,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-12-07T15:16:13,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-07T15:16:13,885 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T15:16:13,886 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T15:16:13,886 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T15:16:13,983 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/C/9dbe03f66c824444bed9465c040bb33c 2024-12-07T15:16:13,987 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/A/9555352250874630bd2b7ab3f9919451 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/9555352250874630bd2b7ab3f9919451 2024-12-07T15:16:13,990 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/9555352250874630bd2b7ab3f9919451, entries=200, sequenceid=114, filesize=38.6 K 2024-12-07T15:16:13,991 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/B/8e11b48d6efe448e966c83661a5579d2 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/8e11b48d6efe448e966c83661a5579d2 2024-12-07T15:16:13,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-07T15:16:13,994 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/8e11b48d6efe448e966c83661a5579d2, entries=150, sequenceid=114, filesize=11.7 K 2024-12-07T15:16:13,995 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/C/9dbe03f66c824444bed9465c040bb33c as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/9dbe03f66c824444bed9465c040bb33c 2024-12-07T15:16:13,998 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/9dbe03f66c824444bed9465c040bb33c, entries=150, sequenceid=114, filesize=11.7 K 2024-12-07T15:16:13,998 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 069dbda53fbb065f68f88519d6f94bc4 in 1662ms, sequenceid=114, compaction requested=true 2024-12-07T15:16:13,999 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 069dbda53fbb065f68f88519d6f94bc4: 2024-12-07T15:16:13,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 069dbda53fbb065f68f88519d6f94bc4:A, priority=-2147483648, current under compaction store size is 1 2024-12-07T15:16:13,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:16:13,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 069dbda53fbb065f68f88519d6f94bc4:B, priority=-2147483648, current under compaction store size is 2 2024-12-07T15:16:13,999 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:16:13,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:16:13,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 069dbda53fbb065f68f88519d6f94bc4:C, priority=-2147483648, current under compaction store size is 3 2024-12-07T15:16:13,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T15:16:13,999 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:16:14,000 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:16:14,000 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:16:14,000 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1541): 069dbda53fbb065f68f88519d6f94bc4/A is initiating minor compaction (all files) 2024-12-07T15:16:14,000 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1541): 069dbda53fbb065f68f88519d6f94bc4/B is initiating minor compaction (all files) 2024-12-07T15:16:14,000 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 069dbda53fbb065f68f88519d6f94bc4/A in TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:14,000 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 069dbda53fbb065f68f88519d6f94bc4/B in TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:14,000 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/aa2b61a68ad344a5b9247a5ddffb8dd9, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/df5054d67b1645de8116f9f6fa90796d, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/9555352250874630bd2b7ab3f9919451] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp, totalSize=99.2 K 2024-12-07T15:16:14,000 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/2a339d6a654e4b86940f554eae331e4e, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/6147dbaaf78e4e03adaa546b103f1273, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/8e11b48d6efe448e966c83661a5579d2] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp, totalSize=35.3 K 2024-12-07T15:16:14,000 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:14,000 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. files: [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/aa2b61a68ad344a5b9247a5ddffb8dd9, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/df5054d67b1645de8116f9f6fa90796d, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/9555352250874630bd2b7ab3f9919451] 2024-12-07T15:16:14,000 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a339d6a654e4b86940f554eae331e4e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733584567757 2024-12-07T15:16:14,001 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 6147dbaaf78e4e03adaa546b103f1273, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733584569001 2024-12-07T15:16:14,001 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa2b61a68ad344a5b9247a5ddffb8dd9, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733584567757 2024-12-07T15:16:14,001 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e11b48d6efe448e966c83661a5579d2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1733584571176 2024-12-07T15:16:14,001 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting df5054d67b1645de8116f9f6fa90796d, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733584569001 2024-12-07T15:16:14,002 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9555352250874630bd2b7ab3f9919451, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1733584571176 2024-12-07T15:16:14,007 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=069dbda53fbb065f68f88519d6f94bc4] 2024-12-07T15:16:14,008 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 069dbda53fbb065f68f88519d6f94bc4#B#compaction#478 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:16:14,008 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/B/7efca4b8647a4590be0eba1441795fb4 is 50, key is test_row_0/B:col10/1733584572335/Put/seqid=0 2024-12-07T15:16:14,009 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241207764dcb5c8385421a9f364c49194cfdf6_069dbda53fbb065f68f88519d6f94bc4 store=[table=TestAcidGuarantees family=A region=069dbda53fbb065f68f88519d6f94bc4] 2024-12-07T15:16:14,011 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241207764dcb5c8385421a9f364c49194cfdf6_069dbda53fbb065f68f88519d6f94bc4, store=[table=TestAcidGuarantees family=A region=069dbda53fbb065f68f88519d6f94bc4] 2024-12-07T15:16:14,012 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207764dcb5c8385421a9f364c49194cfdf6_069dbda53fbb065f68f88519d6f94bc4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=069dbda53fbb065f68f88519d6f94bc4] 2024-12-07T15:16:14,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742386_1562 (size=4469) 2024-12-07T15:16:14,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742385_1561 (size=12207) 2024-12-07T15:16:14,038 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-07T15:16:14,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:14,038 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing 069dbda53fbb065f68f88519d6f94bc4 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-07T15:16:14,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 069dbda53fbb065f68f88519d6f94bc4, store=A 2024-12-07T15:16:14,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:16:14,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 069dbda53fbb065f68f88519d6f94bc4, store=B 2024-12-07T15:16:14,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:16:14,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 069dbda53fbb065f68f88519d6f94bc4, store=C 2024-12-07T15:16:14,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:16:14,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412073d09780822634b6982cbea9a3c9dff00_069dbda53fbb065f68f88519d6f94bc4 is 50, key is test_row_0/A:col10/1733584572373/Put/seqid=0 2024-12-07T15:16:14,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742387_1563 (size=12254) 2024-12-07T15:16:14,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-07T15:16:14,416 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 069dbda53fbb065f68f88519d6f94bc4#A#compaction#479 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:16:14,417 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/A/ea5baf5c16e547129abe2a55b8b8e750 is 175, key is test_row_0/A:col10/1733584572335/Put/seqid=0 2024-12-07T15:16:14,422 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/B/7efca4b8647a4590be0eba1441795fb4 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/7efca4b8647a4590be0eba1441795fb4 2024-12-07T15:16:14,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742388_1564 (size=31161) 2024-12-07T15:16:14,427 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 069dbda53fbb065f68f88519d6f94bc4/B of 069dbda53fbb065f68f88519d6f94bc4 into 7efca4b8647a4590be0eba1441795fb4(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:16:14,427 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 069dbda53fbb065f68f88519d6f94bc4: 2024-12-07T15:16:14,427 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., storeName=069dbda53fbb065f68f88519d6f94bc4/B, priority=13, startTime=1733584573999; duration=0sec 2024-12-07T15:16:14,427 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T15:16:14,427 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 069dbda53fbb065f68f88519d6f94bc4:B 2024-12-07T15:16:14,427 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:16:14,428 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/A/ea5baf5c16e547129abe2a55b8b8e750 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/ea5baf5c16e547129abe2a55b8b8e750 2024-12-07T15:16:14,428 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:16:14,428 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1541): 069dbda53fbb065f68f88519d6f94bc4/C is initiating minor compaction (all files) 2024-12-07T15:16:14,428 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 069dbda53fbb065f68f88519d6f94bc4/C in TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:14,429 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/bf759a03a63644ce8bbeace14a4c8b58, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/3c99821889c24a41b4c0e1662092eec9, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/9dbe03f66c824444bed9465c040bb33c] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp, totalSize=35.3 K 2024-12-07T15:16:14,429 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting bf759a03a63644ce8bbeace14a4c8b58, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733584567757 2024-12-07T15:16:14,429 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c99821889c24a41b4c0e1662092eec9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733584569001 2024-12-07T15:16:14,429 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 9dbe03f66c824444bed9465c040bb33c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1733584571176 2024-12-07T15:16:14,432 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 069dbda53fbb065f68f88519d6f94bc4/A of 069dbda53fbb065f68f88519d6f94bc4 into ea5baf5c16e547129abe2a55b8b8e750(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:16:14,432 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 069dbda53fbb065f68f88519d6f94bc4: 2024-12-07T15:16:14,432 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., storeName=069dbda53fbb065f68f88519d6f94bc4/A, priority=13, startTime=1733584573999; duration=0sec 2024-12-07T15:16:14,432 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:16:14,433 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 069dbda53fbb065f68f88519d6f94bc4:A 2024-12-07T15:16:14,438 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 069dbda53fbb065f68f88519d6f94bc4#C#compaction#481 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:16:14,438 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/C/e5355ae86dcb4e559c62e86a432236ea is 50, key is test_row_0/C:col10/1733584572335/Put/seqid=0 2024-12-07T15:16:14,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742389_1565 (size=12207) 2024-12-07T15:16:14,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:16:14,452 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412073d09780822634b6982cbea9a3c9dff00_069dbda53fbb065f68f88519d6f94bc4 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412073d09780822634b6982cbea9a3c9dff00_069dbda53fbb065f68f88519d6f94bc4 2024-12-07T15:16:14,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/A/e33eeb0e0fcc42e79c7b593df3f57b7d, store: [table=TestAcidGuarantees family=A region=069dbda53fbb065f68f88519d6f94bc4] 2024-12-07T15:16:14,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/A/e33eeb0e0fcc42e79c7b593df3f57b7d is 175, key is test_row_0/A:col10/1733584572373/Put/seqid=0 2024-12-07T15:16:14,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742390_1566 (size=31055) 2024-12-07T15:16:14,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-07T15:16:14,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on 069dbda53fbb065f68f88519d6f94bc4 2024-12-07T15:16:14,526 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. as already flushing 2024-12-07T15:16:14,563 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:14,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44662 deadline: 1733584634558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:14,563 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:14,563 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:14,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44694 deadline: 1733584634559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:14,563 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:14,563 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:14,563 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:14,563 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:14,563 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:14,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:14,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44662 deadline: 1733584634673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:14,674 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:14,674 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:14,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44694 deadline: 1733584634673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:14,674 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:14,674 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:14,674 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:14,674 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:14,674 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:14,846 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/C/e5355ae86dcb4e559c62e86a432236ea as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/e5355ae86dcb4e559c62e86a432236ea 2024-12-07T15:16:14,850 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 069dbda53fbb065f68f88519d6f94bc4/C of 069dbda53fbb065f68f88519d6f94bc4 into e5355ae86dcb4e559c62e86a432236ea(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:16:14,850 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 069dbda53fbb065f68f88519d6f94bc4: 2024-12-07T15:16:14,850 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., storeName=069dbda53fbb065f68f88519d6f94bc4/C, priority=13, startTime=1733584573999; duration=0sec 2024-12-07T15:16:14,850 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:16:14,850 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 069dbda53fbb065f68f88519d6f94bc4:C 2024-12-07T15:16:14,859 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=133, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/A/e33eeb0e0fcc42e79c7b593df3f57b7d 2024-12-07T15:16:14,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/B/0d94443d2dc04c498f00a1894a810788 is 50, key is test_row_0/B:col10/1733584572373/Put/seqid=0 2024-12-07T15:16:14,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742391_1567 (size=12101) 2024-12-07T15:16:14,887 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:14,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44662 deadline: 1733584634883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:14,887 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:14,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44694 deadline: 1733584634884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:14,887 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:14,888 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:14,888 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:14,888 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:14,888 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:14,888 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:15,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-07T15:16:15,196 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:15,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44662 deadline: 1733584635193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:15,196 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:15,196 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:15,196 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:15,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44694 deadline: 1733584635193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:15,196 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:15,197 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:15,197 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:15,197 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:15,268 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/B/0d94443d2dc04c498f00a1894a810788 2024-12-07T15:16:15,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/C/f3235915c4e94cb2b4eb1a98af2dfd87 is 50, key is test_row_0/C:col10/1733584572373/Put/seqid=0 2024-12-07T15:16:15,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742392_1568 (size=12101) 2024-12-07T15:16:15,679 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/C/f3235915c4e94cb2b4eb1a98af2dfd87 2024-12-07T15:16:15,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/A/e33eeb0e0fcc42e79c7b593df3f57b7d as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/e33eeb0e0fcc42e79c7b593df3f57b7d 2024-12-07T15:16:15,685 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/e33eeb0e0fcc42e79c7b593df3f57b7d, entries=150, sequenceid=133, filesize=30.3 K 2024-12-07T15:16:15,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/B/0d94443d2dc04c498f00a1894a810788 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/0d94443d2dc04c498f00a1894a810788 2024-12-07T15:16:15,689 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/0d94443d2dc04c498f00a1894a810788, entries=150, sequenceid=133, filesize=11.8 K 2024-12-07T15:16:15,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/C/f3235915c4e94cb2b4eb1a98af2dfd87 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/f3235915c4e94cb2b4eb1a98af2dfd87 2024-12-07T15:16:15,692 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/f3235915c4e94cb2b4eb1a98af2dfd87, entries=150, sequenceid=133, filesize=11.8 K 2024-12-07T15:16:15,693 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for 069dbda53fbb065f68f88519d6f94bc4 in 1655ms, sequenceid=133, compaction requested=false 2024-12-07T15:16:15,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for 069dbda53fbb065f68f88519d6f94bc4: 2024-12-07T15:16:15,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:15,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-12-07T15:16:15,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-12-07T15:16:15,697 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-12-07T15:16:15,697 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8090 sec 2024-12-07T15:16:15,699 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 1.8150 sec 2024-12-07T15:16:15,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on 069dbda53fbb065f68f88519d6f94bc4 2024-12-07T15:16:15,706 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 069dbda53fbb065f68f88519d6f94bc4 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-07T15:16:15,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 069dbda53fbb065f68f88519d6f94bc4, store=A 2024-12-07T15:16:15,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:16:15,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 069dbda53fbb065f68f88519d6f94bc4, store=B 2024-12-07T15:16:15,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:16:15,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 069dbda53fbb065f68f88519d6f94bc4, store=C 2024-12-07T15:16:15,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:16:15,719 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120735dbfde0016c41c6ac8caae2249c66ed_069dbda53fbb065f68f88519d6f94bc4 is 50, key is test_row_0/A:col10/1733584574557/Put/seqid=0 2024-12-07T15:16:15,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742393_1569 (size=12304) 2024-12-07T15:16:15,742 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:15,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44694 deadline: 1733584635738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:15,742 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:15,742 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:15,742 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:15,745 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:15,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44662 deadline: 1733584635741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:15,746 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:15,746 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:15,746 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:15,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:15,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44694 deadline: 1733584635853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:15,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:15,857 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:15,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44662 deadline: 1733584635853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:15,857 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:15,857 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:15,857 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:15,857 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:15,857 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:16,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-07T15:16:16,034 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T15:16:16,034 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T15:16:16,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees 2024-12-07T15:16:16,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-07T15:16:16,036 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T15:16:16,036 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T15:16:16,036 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T15:16:16,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:16,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44694 deadline: 1733584636063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:16,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:16,067 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:16,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44662 deadline: 1733584636063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:16,067 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:16,067 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:16,067 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:16,067 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:16,067 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:16,123 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:16:16,126 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120735dbfde0016c41c6ac8caae2249c66ed_069dbda53fbb065f68f88519d6f94bc4 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120735dbfde0016c41c6ac8caae2249c66ed_069dbda53fbb065f68f88519d6f94bc4 2024-12-07T15:16:16,126 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/A/c545dc54d2954d7883a655303693c480, store: [table=TestAcidGuarantees family=A region=069dbda53fbb065f68f88519d6f94bc4] 2024-12-07T15:16:16,127 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/A/c545dc54d2954d7883a655303693c480 is 175, key is test_row_0/A:col10/1733584574557/Put/seqid=0 2024-12-07T15:16:16,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742394_1570 (size=31105) 2024-12-07T15:16:16,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-07T15:16:16,188 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-07T15:16:16,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:16,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. as already flushing 2024-12-07T15:16:16,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:16,189 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:16:16,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:16:16,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:16:16,341 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-07T15:16:16,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:16,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. as already flushing 2024-12-07T15:16:16,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:16,342 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:16:16,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:16:16,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:16:16,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-07T15:16:16,374 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:16,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44694 deadline: 1733584636374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:16,375 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:16,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:16,375 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:16,375 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:16,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44662 deadline: 1733584636374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:16,375 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:16,375 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:16,375 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:16,494 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-07T15:16:16,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:16,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. as already flushing 2024-12-07T15:16:16,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:16,494 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:16:16,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:16:16,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:16:16,533 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=151, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/A/c545dc54d2954d7883a655303693c480 2024-12-07T15:16:16,540 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/B/94b3d8797c8b4878acc10d50f2617898 is 50, key is test_row_0/B:col10/1733584574557/Put/seqid=0 2024-12-07T15:16:16,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742395_1571 (size=12151) 2024-12-07T15:16:16,647 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-07T15:16:16,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:16,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. as already flushing 2024-12-07T15:16:16,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:16,647 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:16:16,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:16:16,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:16:16,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-07T15:16:16,800 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-07T15:16:16,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:16,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. as already flushing 2024-12-07T15:16:16,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:16,800 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:16:16,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:16:16,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:16:16,887 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:16,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44694 deadline: 1733584636883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:16,887 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:16,887 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:16,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44662 deadline: 1733584636883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:16,887 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:16,887 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:16,887 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:16,887 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:16,887 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:16,944 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/B/94b3d8797c8b4878acc10d50f2617898 2024-12-07T15:16:16,951 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/C/8360848629f44b259dedc3fa0e8b9183 is 50, key is test_row_0/C:col10/1733584574557/Put/seqid=0 2024-12-07T15:16:16,952 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-07T15:16:16,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:16,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. as already flushing 2024-12-07T15:16:16,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:16,953 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:16:16,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:16:16,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:16:16,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742396_1572 (size=12151) 2024-12-07T15:16:16,956 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/C/8360848629f44b259dedc3fa0e8b9183 2024-12-07T15:16:16,960 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/A/c545dc54d2954d7883a655303693c480 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/c545dc54d2954d7883a655303693c480 2024-12-07T15:16:16,962 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/c545dc54d2954d7883a655303693c480, entries=150, sequenceid=151, filesize=30.4 K 2024-12-07T15:16:16,963 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/B/94b3d8797c8b4878acc10d50f2617898 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/94b3d8797c8b4878acc10d50f2617898 2024-12-07T15:16:16,966 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/94b3d8797c8b4878acc10d50f2617898, entries=150, sequenceid=151, filesize=11.9 K 2024-12-07T15:16:16,966 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/C/8360848629f44b259dedc3fa0e8b9183 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/8360848629f44b259dedc3fa0e8b9183 2024-12-07T15:16:16,969 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/8360848629f44b259dedc3fa0e8b9183, entries=150, sequenceid=151, filesize=11.9 K 2024-12-07T15:16:16,970 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 069dbda53fbb065f68f88519d6f94bc4 in 1264ms, sequenceid=151, compaction requested=true 2024-12-07T15:16:16,970 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 069dbda53fbb065f68f88519d6f94bc4: 2024-12-07T15:16:16,970 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 069dbda53fbb065f68f88519d6f94bc4:A, priority=-2147483648, current under compaction store size is 1 2024-12-07T15:16:16,970 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:16:16,970 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:16:16,970 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 069dbda53fbb065f68f88519d6f94bc4:B, priority=-2147483648, current under compaction store size is 2 2024-12-07T15:16:16,970 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:16:16,970 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 069dbda53fbb065f68f88519d6f94bc4:C, priority=-2147483648, current under compaction store size is 3 2024-12-07T15:16:16,970 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T15:16:16,970 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:16:16,971 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93321 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:16:16,971 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1541): 069dbda53fbb065f68f88519d6f94bc4/A is initiating minor compaction (all files) 2024-12-07T15:16:16,971 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 069dbda53fbb065f68f88519d6f94bc4/A in TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:16,971 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/ea5baf5c16e547129abe2a55b8b8e750, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/e33eeb0e0fcc42e79c7b593df3f57b7d, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/c545dc54d2954d7883a655303693c480] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp, totalSize=91.1 K 2024-12-07T15:16:16,971 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:16,971 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. files: [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/ea5baf5c16e547129abe2a55b8b8e750, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/e33eeb0e0fcc42e79c7b593df3f57b7d, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/c545dc54d2954d7883a655303693c480] 2024-12-07T15:16:16,971 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:16:16,971 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1541): 069dbda53fbb065f68f88519d6f94bc4/B is initiating minor compaction (all files) 2024-12-07T15:16:16,972 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 069dbda53fbb065f68f88519d6f94bc4/B in TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:16,972 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/7efca4b8647a4590be0eba1441795fb4, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/0d94443d2dc04c498f00a1894a810788, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/94b3d8797c8b4878acc10d50f2617898] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp, totalSize=35.6 K 2024-12-07T15:16:16,972 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea5baf5c16e547129abe2a55b8b8e750, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1733584571176 2024-12-07T15:16:16,972 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 7efca4b8647a4590be0eba1441795fb4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1733584571176 2024-12-07T15:16:16,973 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting e33eeb0e0fcc42e79c7b593df3f57b7d, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1733584572356 2024-12-07T15:16:16,973 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting c545dc54d2954d7883a655303693c480, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1733584574551 2024-12-07T15:16:16,973 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d94443d2dc04c498f00a1894a810788, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1733584572356 2024-12-07T15:16:16,973 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 94b3d8797c8b4878acc10d50f2617898, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1733584574551 2024-12-07T15:16:16,981 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 069dbda53fbb065f68f88519d6f94bc4#B#compaction#487 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:16:16,981 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/B/637f2b0698284c5fbb5a1e20fd6ac301 is 50, key is test_row_0/B:col10/1733584574557/Put/seqid=0 2024-12-07T15:16:16,989 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=069dbda53fbb065f68f88519d6f94bc4] 2024-12-07T15:16:16,991 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241207ec57a4e18fff46138acf5ea0566c1f12_069dbda53fbb065f68f88519d6f94bc4 store=[table=TestAcidGuarantees family=A region=069dbda53fbb065f68f88519d6f94bc4] 2024-12-07T15:16:16,993 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241207ec57a4e18fff46138acf5ea0566c1f12_069dbda53fbb065f68f88519d6f94bc4, store=[table=TestAcidGuarantees family=A region=069dbda53fbb065f68f88519d6f94bc4] 2024-12-07T15:16:16,993 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207ec57a4e18fff46138acf5ea0566c1f12_069dbda53fbb065f68f88519d6f94bc4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=069dbda53fbb065f68f88519d6f94bc4] 2024-12-07T15:16:17,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742397_1573 (size=12459) 2024-12-07T15:16:17,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742398_1574 (size=4469) 2024-12-07T15:16:17,105 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-07T15:16:17,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:17,106 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2902): Flushing 069dbda53fbb065f68f88519d6f94bc4 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-07T15:16:17,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 069dbda53fbb065f68f88519d6f94bc4, store=A 2024-12-07T15:16:17,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:16:17,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 069dbda53fbb065f68f88519d6f94bc4, store=B 2024-12-07T15:16:17,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:16:17,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 069dbda53fbb065f68f88519d6f94bc4, store=C 2024-12-07T15:16:17,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:16:17,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207ca169a7c874b43dfaea588bc3abcf307_069dbda53fbb065f68f88519d6f94bc4 is 50, key is test_row_0/A:col10/1733584575732/Put/seqid=0 2024-12-07T15:16:17,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742399_1575 (size=12304) 2024-12-07T15:16:17,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-07T15:16:17,403 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 069dbda53fbb065f68f88519d6f94bc4#A#compaction#488 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:16:17,404 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/A/f3a899119ce84389843d56a6633adadf is 175, key is test_row_0/A:col10/1733584574557/Put/seqid=0 2024-12-07T15:16:17,405 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/B/637f2b0698284c5fbb5a1e20fd6ac301 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/637f2b0698284c5fbb5a1e20fd6ac301 2024-12-07T15:16:17,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742400_1576 (size=31413) 2024-12-07T15:16:17,411 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 069dbda53fbb065f68f88519d6f94bc4/B of 069dbda53fbb065f68f88519d6f94bc4 into 637f2b0698284c5fbb5a1e20fd6ac301(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:16:17,411 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 069dbda53fbb065f68f88519d6f94bc4: 2024-12-07T15:16:17,411 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., storeName=069dbda53fbb065f68f88519d6f94bc4/B, priority=13, startTime=1733584576970; duration=0sec 2024-12-07T15:16:17,411 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T15:16:17,411 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 069dbda53fbb065f68f88519d6f94bc4:B 2024-12-07T15:16:17,411 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:16:17,412 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:16:17,412 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1541): 069dbda53fbb065f68f88519d6f94bc4/C is initiating minor compaction (all files) 2024-12-07T15:16:17,412 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 069dbda53fbb065f68f88519d6f94bc4/C in TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:17,413 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/e5355ae86dcb4e559c62e86a432236ea, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/f3235915c4e94cb2b4eb1a98af2dfd87, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/8360848629f44b259dedc3fa0e8b9183] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp, totalSize=35.6 K 2024-12-07T15:16:17,413 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting e5355ae86dcb4e559c62e86a432236ea, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1733584571176 2024-12-07T15:16:17,413 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting f3235915c4e94cb2b4eb1a98af2dfd87, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1733584572356 2024-12-07T15:16:17,413 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 8360848629f44b259dedc3fa0e8b9183, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1733584574551 2024-12-07T15:16:17,420 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 069dbda53fbb065f68f88519d6f94bc4#C#compaction#490 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:16:17,420 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/C/f0f9bf5ef8f84566a416549159c55b3f is 50, key is test_row_0/C:col10/1733584574557/Put/seqid=0 2024-12-07T15:16:17,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742401_1577 (size=12459) 2024-12-07T15:16:17,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:16:17,520 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207ca169a7c874b43dfaea588bc3abcf307_069dbda53fbb065f68f88519d6f94bc4 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207ca169a7c874b43dfaea588bc3abcf307_069dbda53fbb065f68f88519d6f94bc4 2024-12-07T15:16:17,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/A/b538aa39a4b04ab8a3e6dcf25a8f3567, store: [table=TestAcidGuarantees family=A region=069dbda53fbb065f68f88519d6f94bc4] 2024-12-07T15:16:17,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/A/b538aa39a4b04ab8a3e6dcf25a8f3567 is 175, key is test_row_0/A:col10/1733584575732/Put/seqid=0 2024-12-07T15:16:17,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742402_1578 (size=31105) 2024-12-07T15:16:17,525 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=168, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/A/b538aa39a4b04ab8a3e6dcf25a8f3567 2024-12-07T15:16:17,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/B/2f2bb04f49dc48719ae99e340402a33a is 50, key is test_row_0/B:col10/1733584575732/Put/seqid=0 2024-12-07T15:16:17,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742403_1579 (size=12151) 2024-12-07T15:16:17,813 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/A/f3a899119ce84389843d56a6633adadf as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/f3a899119ce84389843d56a6633adadf 2024-12-07T15:16:17,817 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 069dbda53fbb065f68f88519d6f94bc4/A of 069dbda53fbb065f68f88519d6f94bc4 into f3a899119ce84389843d56a6633adadf(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:16:17,817 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 069dbda53fbb065f68f88519d6f94bc4: 2024-12-07T15:16:17,817 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., storeName=069dbda53fbb065f68f88519d6f94bc4/A, priority=13, startTime=1733584576970; duration=0sec 2024-12-07T15:16:17,818 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:16:17,818 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 069dbda53fbb065f68f88519d6f94bc4:A 2024-12-07T15:16:17,836 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/C/f0f9bf5ef8f84566a416549159c55b3f as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/f0f9bf5ef8f84566a416549159c55b3f 2024-12-07T15:16:17,841 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 069dbda53fbb065f68f88519d6f94bc4/C of 069dbda53fbb065f68f88519d6f94bc4 into f0f9bf5ef8f84566a416549159c55b3f(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:16:17,841 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 069dbda53fbb065f68f88519d6f94bc4: 2024-12-07T15:16:17,841 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., storeName=069dbda53fbb065f68f88519d6f94bc4/C, priority=13, startTime=1733584576970; duration=0sec 2024-12-07T15:16:17,841 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:16:17,841 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 069dbda53fbb065f68f88519d6f94bc4:C 2024-12-07T15:16:17,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on 069dbda53fbb065f68f88519d6f94bc4 2024-12-07T15:16:17,906 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. as already flushing 2024-12-07T15:16:17,936 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/B/2f2bb04f49dc48719ae99e340402a33a 2024-12-07T15:16:17,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:17,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44662 deadline: 1733584637932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:17,938 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:17,938 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:17,938 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:17,938 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:17,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44694 deadline: 1733584637933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:17,938 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:17,938 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:17,939 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:17,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/C/8125d0e876ae445390a4cbf68416d7fc is 50, key is test_row_0/C:col10/1733584575732/Put/seqid=0 2024-12-07T15:16:17,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742404_1580 (size=12151) 2024-12-07T15:16:18,046 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:18,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44662 deadline: 1733584638043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:18,046 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:18,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44694 deadline: 1733584638043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:18,046 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:18,046 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:18,046 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:18,046 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:18,046 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:18,046 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:18,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-07T15:16:18,256 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:18,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44662 deadline: 1733584638253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:18,256 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:18,256 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:18,256 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:18,256 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:18,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44694 deadline: 1733584638253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:18,257 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:18,257 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:18,257 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:18,346 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/C/8125d0e876ae445390a4cbf68416d7fc 2024-12-07T15:16:18,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/A/b538aa39a4b04ab8a3e6dcf25a8f3567 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/b538aa39a4b04ab8a3e6dcf25a8f3567 2024-12-07T15:16:18,353 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/b538aa39a4b04ab8a3e6dcf25a8f3567, entries=150, sequenceid=168, filesize=30.4 K 2024-12-07T15:16:18,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/B/2f2bb04f49dc48719ae99e340402a33a as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/2f2bb04f49dc48719ae99e340402a33a 2024-12-07T15:16:18,357 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/2f2bb04f49dc48719ae99e340402a33a, entries=150, sequenceid=168, filesize=11.9 K 2024-12-07T15:16:18,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/C/8125d0e876ae445390a4cbf68416d7fc as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/8125d0e876ae445390a4cbf68416d7fc 2024-12-07T15:16:18,361 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/8125d0e876ae445390a4cbf68416d7fc, entries=150, sequenceid=168, filesize=11.9 K 2024-12-07T15:16:18,361 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 069dbda53fbb065f68f88519d6f94bc4 in 1255ms, sequenceid=168, compaction requested=false 2024-12-07T15:16:18,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2603): Flush status journal for 069dbda53fbb065f68f88519d6f94bc4: 2024-12-07T15:16:18,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:18,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=165 2024-12-07T15:16:18,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4169): Remote procedure done, pid=165 2024-12-07T15:16:18,364 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-12-07T15:16:18,364 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3270 sec 2024-12-07T15:16:18,366 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees in 2.3310 sec 2024-12-07T15:16:18,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on 069dbda53fbb065f68f88519d6f94bc4 2024-12-07T15:16:18,566 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 069dbda53fbb065f68f88519d6f94bc4 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-07T15:16:18,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 069dbda53fbb065f68f88519d6f94bc4, store=A 2024-12-07T15:16:18,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:16:18,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 069dbda53fbb065f68f88519d6f94bc4, store=B 2024-12-07T15:16:18,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:16:18,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 069dbda53fbb065f68f88519d6f94bc4, store=C 2024-12-07T15:16:18,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:16:18,573 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207b7f114baf3394b718095824ad6850079_069dbda53fbb065f68f88519d6f94bc4 is 50, key is test_row_0/A:col10/1733584577930/Put/seqid=0 2024-12-07T15:16:18,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742405_1581 (size=12304) 2024-12-07T15:16:18,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:18,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44662 deadline: 1733584638591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:18,596 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:18,596 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:18,596 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:18,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:18,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44694 deadline: 1733584638596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:18,602 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:18,602 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:18,602 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:18,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:18,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44662 deadline: 1733584638703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:18,706 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:18,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:18,706 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:18,706 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:18,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44694 deadline: 1733584638703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:18,707 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:18,707 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:18,707 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:18,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:18,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44662 deadline: 1733584638913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:18,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:18,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44694 deadline: 1733584638914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:18,917 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:18,917 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:18,917 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:18,918 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:18,918 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:18,918 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:18,984 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:16:18,987 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207b7f114baf3394b718095824ad6850079_069dbda53fbb065f68f88519d6f94bc4 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207b7f114baf3394b718095824ad6850079_069dbda53fbb065f68f88519d6f94bc4 2024-12-07T15:16:18,988 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/A/a53bd14d01634c3d9932dddde412d364, store: [table=TestAcidGuarantees family=A region=069dbda53fbb065f68f88519d6f94bc4] 2024-12-07T15:16:18,988 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/A/a53bd14d01634c3d9932dddde412d364 is 175, key is test_row_0/A:col10/1733584577930/Put/seqid=0 2024-12-07T15:16:18,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742406_1582 (size=31105) 2024-12-07T15:16:19,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:19,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44662 deadline: 1733584639223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:19,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:19,226 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:19,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44694 deadline: 1733584639223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:19,227 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:19,227 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:19,227 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:19,227 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:19,227 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:19,392 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=188, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/A/a53bd14d01634c3d9932dddde412d364 2024-12-07T15:16:19,399 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/B/c9c7be4bf0644dceb6e332780e8315ee is 50, key is test_row_0/B:col10/1733584577930/Put/seqid=0 2024-12-07T15:16:19,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742407_1583 (size=12151) 2024-12-07T15:16:19,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:19,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44662 deadline: 1733584639733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:19,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:19,735 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:19,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44694 deadline: 1733584639733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:19,736 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:19,736 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:19,736 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:19,736 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:19,736 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:19,803 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/B/c9c7be4bf0644dceb6e332780e8315ee 2024-12-07T15:16:19,810 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/C/c5ccf9755f644ac29d9cf2e061d5993a is 50, key is test_row_0/C:col10/1733584577930/Put/seqid=0 2024-12-07T15:16:19,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742408_1584 (size=12151) 2024-12-07T15:16:19,813 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/C/c5ccf9755f644ac29d9cf2e061d5993a 2024-12-07T15:16:19,816 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/A/a53bd14d01634c3d9932dddde412d364 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/a53bd14d01634c3d9932dddde412d364 2024-12-07T15:16:19,819 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/a53bd14d01634c3d9932dddde412d364, entries=150, sequenceid=188, filesize=30.4 K 2024-12-07T15:16:19,820 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/B/c9c7be4bf0644dceb6e332780e8315ee as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/c9c7be4bf0644dceb6e332780e8315ee 2024-12-07T15:16:19,823 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/c9c7be4bf0644dceb6e332780e8315ee, entries=150, sequenceid=188, filesize=11.9 K 2024-12-07T15:16:19,823 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/C/c5ccf9755f644ac29d9cf2e061d5993a as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/c5ccf9755f644ac29d9cf2e061d5993a 2024-12-07T15:16:19,828 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/c5ccf9755f644ac29d9cf2e061d5993a, entries=150, sequenceid=188, filesize=11.9 K 2024-12-07T15:16:19,829 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 069dbda53fbb065f68f88519d6f94bc4 in 1264ms, sequenceid=188, compaction requested=true 2024-12-07T15:16:19,829 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 069dbda53fbb065f68f88519d6f94bc4: 2024-12-07T15:16:19,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 069dbda53fbb065f68f88519d6f94bc4:A, priority=-2147483648, current under compaction store size is 1 2024-12-07T15:16:19,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:16:19,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 069dbda53fbb065f68f88519d6f94bc4:B, priority=-2147483648, current under compaction store size is 2 2024-12-07T15:16:19,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:16:19,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 069dbda53fbb065f68f88519d6f94bc4:C, priority=-2147483648, current under compaction store size is 3 2024-12-07T15:16:19,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T15:16:19,829 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:16:19,829 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:16:19,830 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:16:19,830 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1541): 069dbda53fbb065f68f88519d6f94bc4/B is initiating minor compaction (all files) 2024-12-07T15:16:19,830 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93623 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:16:19,830 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 069dbda53fbb065f68f88519d6f94bc4/B in TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:19,830 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1541): 069dbda53fbb065f68f88519d6f94bc4/A is initiating minor compaction (all files) 2024-12-07T15:16:19,830 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/637f2b0698284c5fbb5a1e20fd6ac301, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/2f2bb04f49dc48719ae99e340402a33a, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/c9c7be4bf0644dceb6e332780e8315ee] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp, totalSize=35.9 K 2024-12-07T15:16:19,830 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 069dbda53fbb065f68f88519d6f94bc4/A in TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:19,830 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/f3a899119ce84389843d56a6633adadf, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/b538aa39a4b04ab8a3e6dcf25a8f3567, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/a53bd14d01634c3d9932dddde412d364] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp, totalSize=91.4 K 2024-12-07T15:16:19,830 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:19,830 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. files: [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/f3a899119ce84389843d56a6633adadf, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/b538aa39a4b04ab8a3e6dcf25a8f3567, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/a53bd14d01634c3d9932dddde412d364] 2024-12-07T15:16:19,830 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 637f2b0698284c5fbb5a1e20fd6ac301, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1733584574551 2024-12-07T15:16:19,831 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting f3a899119ce84389843d56a6633adadf, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1733584574551 2024-12-07T15:16:19,831 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f2bb04f49dc48719ae99e340402a33a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1733584575732 2024-12-07T15:16:19,831 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting b538aa39a4b04ab8a3e6dcf25a8f3567, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1733584575732 2024-12-07T15:16:19,831 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting c9c7be4bf0644dceb6e332780e8315ee, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1733584577930 2024-12-07T15:16:19,831 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting a53bd14d01634c3d9932dddde412d364, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1733584577930 2024-12-07T15:16:19,837 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=069dbda53fbb065f68f88519d6f94bc4] 2024-12-07T15:16:19,838 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 069dbda53fbb065f68f88519d6f94bc4#B#compaction#496 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:16:19,838 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/B/f18c129635714bac831c83be303cac5c is 50, key is test_row_0/B:col10/1733584577930/Put/seqid=0 2024-12-07T15:16:19,839 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241207b946ebde3a9748c8868c5e783c95a8b8_069dbda53fbb065f68f88519d6f94bc4 store=[table=TestAcidGuarantees family=A region=069dbda53fbb065f68f88519d6f94bc4] 2024-12-07T15:16:19,841 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241207b946ebde3a9748c8868c5e783c95a8b8_069dbda53fbb065f68f88519d6f94bc4, store=[table=TestAcidGuarantees family=A region=069dbda53fbb065f68f88519d6f94bc4] 2024-12-07T15:16:19,841 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207b946ebde3a9748c8868c5e783c95a8b8_069dbda53fbb065f68f88519d6f94bc4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=069dbda53fbb065f68f88519d6f94bc4] 2024-12-07T15:16:19,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742409_1585 (size=12561) 2024-12-07T15:16:19,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742410_1586 (size=4469) 2024-12-07T15:16:19,846 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 069dbda53fbb065f68f88519d6f94bc4#A#compaction#497 average throughput is 2.71 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:16:19,847 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/A/e04d234b06fd429d89a2a7c6d92f0e84 is 175, key is test_row_0/A:col10/1733584577930/Put/seqid=0 2024-12-07T15:16:19,848 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/B/f18c129635714bac831c83be303cac5c as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/f18c129635714bac831c83be303cac5c 2024-12-07T15:16:19,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742411_1587 (size=31515) 2024-12-07T15:16:19,853 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 069dbda53fbb065f68f88519d6f94bc4/B of 069dbda53fbb065f68f88519d6f94bc4 into f18c129635714bac831c83be303cac5c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:16:19,853 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 069dbda53fbb065f68f88519d6f94bc4: 2024-12-07T15:16:19,853 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., storeName=069dbda53fbb065f68f88519d6f94bc4/B, priority=13, startTime=1733584579829; duration=0sec 2024-12-07T15:16:19,853 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T15:16:19,853 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 069dbda53fbb065f68f88519d6f94bc4:B 2024-12-07T15:16:19,853 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:16:19,854 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:16:19,854 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1541): 069dbda53fbb065f68f88519d6f94bc4/C is initiating minor compaction (all files) 2024-12-07T15:16:19,854 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 069dbda53fbb065f68f88519d6f94bc4/C in TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:19,854 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/f0f9bf5ef8f84566a416549159c55b3f, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/8125d0e876ae445390a4cbf68416d7fc, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/c5ccf9755f644ac29d9cf2e061d5993a] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp, totalSize=35.9 K 2024-12-07T15:16:19,854 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting f0f9bf5ef8f84566a416549159c55b3f, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1733584574551 2024-12-07T15:16:19,855 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 8125d0e876ae445390a4cbf68416d7fc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1733584575732 2024-12-07T15:16:19,855 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/A/e04d234b06fd429d89a2a7c6d92f0e84 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/e04d234b06fd429d89a2a7c6d92f0e84 2024-12-07T15:16:19,855 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting c5ccf9755f644ac29d9cf2e061d5993a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1733584577930 2024-12-07T15:16:19,858 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 069dbda53fbb065f68f88519d6f94bc4/A of 069dbda53fbb065f68f88519d6f94bc4 into e04d234b06fd429d89a2a7c6d92f0e84(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:16:19,858 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 069dbda53fbb065f68f88519d6f94bc4: 2024-12-07T15:16:19,859 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., storeName=069dbda53fbb065f68f88519d6f94bc4/A, priority=13, startTime=1733584579829; duration=0sec 2024-12-07T15:16:19,859 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:16:19,859 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 069dbda53fbb065f68f88519d6f94bc4:A 2024-12-07T15:16:19,860 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 069dbda53fbb065f68f88519d6f94bc4#C#compaction#498 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:16:19,860 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/C/5a91b0641da04d5b9a1ee6718f29139f is 50, key is test_row_0/C:col10/1733584577930/Put/seqid=0 2024-12-07T15:16:19,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742412_1588 (size=12561) 2024-12-07T15:16:19,872 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/C/5a91b0641da04d5b9a1ee6718f29139f as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/5a91b0641da04d5b9a1ee6718f29139f 2024-12-07T15:16:19,875 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 069dbda53fbb065f68f88519d6f94bc4/C of 069dbda53fbb065f68f88519d6f94bc4 into 5a91b0641da04d5b9a1ee6718f29139f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:16:19,875 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 069dbda53fbb065f68f88519d6f94bc4: 2024-12-07T15:16:19,875 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., storeName=069dbda53fbb065f68f88519d6f94bc4/C, priority=13, startTime=1733584579829; duration=0sec 2024-12-07T15:16:19,875 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:16:19,875 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 069dbda53fbb065f68f88519d6f94bc4:C 2024-12-07T15:16:20,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-07T15:16:20,194 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T15:16:20,194 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T15:16:20,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees 2024-12-07T15:16:20,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-07T15:16:20,196 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T15:16:20,196 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T15:16:20,196 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T15:16:20,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-07T15:16:20,348 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-12-07T15:16:20,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:20,349 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2902): Flushing 069dbda53fbb065f68f88519d6f94bc4 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-07T15:16:20,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 069dbda53fbb065f68f88519d6f94bc4, store=A 2024-12-07T15:16:20,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:16:20,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 069dbda53fbb065f68f88519d6f94bc4, store=B 2024-12-07T15:16:20,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:16:20,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 069dbda53fbb065f68f88519d6f94bc4, store=C 2024-12-07T15:16:20,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:16:20,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412075d3289f000714ce3ad5581f029dc9931_069dbda53fbb065f68f88519d6f94bc4 is 50, key is test_row_0/A:col10/1733584578590/Put/seqid=0 2024-12-07T15:16:20,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742413_1589 (size=12304) 2024-12-07T15:16:20,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-07T15:16:20,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on 069dbda53fbb065f68f88519d6f94bc4 2024-12-07T15:16:20,745 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. as already flushing 2024-12-07T15:16:20,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:16:20,762 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412075d3289f000714ce3ad5581f029dc9931_069dbda53fbb065f68f88519d6f94bc4 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412075d3289f000714ce3ad5581f029dc9931_069dbda53fbb065f68f88519d6f94bc4 2024-12-07T15:16:20,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/A/bec8e521b49640b8b6249b6755c11ce8, store: [table=TestAcidGuarantees family=A region=069dbda53fbb065f68f88519d6f94bc4] 2024-12-07T15:16:20,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/A/bec8e521b49640b8b6249b6755c11ce8 is 175, key is test_row_0/A:col10/1733584578590/Put/seqid=0 2024-12-07T15:16:20,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742414_1590 (size=31105) 2024-12-07T15:16:20,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:20,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44694 deadline: 1733584640776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:20,780 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:20,780 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:20,780 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:20,781 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:20,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44662 deadline: 1733584640778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:20,781 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:20,781 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:20,781 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:20,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-07T15:16:20,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:20,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44694 deadline: 1733584640883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:20,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:20,885 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:20,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44662 deadline: 1733584640883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:20,885 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:20,885 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:20,885 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:20,886 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:20,886 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:21,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:21,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44694 deadline: 1733584641093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:21,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:21,097 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:21,097 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:21,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44662 deadline: 1733584641094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:21,097 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:21,097 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:21,097 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:21,097 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:21,167 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=206, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/A/bec8e521b49640b8b6249b6755c11ce8 2024-12-07T15:16:21,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/B/0556b5f672434a4eb85281e93ffa0c4f is 50, key is test_row_0/B:col10/1733584578590/Put/seqid=0 2024-12-07T15:16:21,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742415_1591 (size=12151) 2024-12-07T15:16:21,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-07T15:16:21,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:21,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44662 deadline: 1733584641403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:21,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:21,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44694 deadline: 1733584641403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:21,407 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:21,407 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:21,407 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:21,407 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:21,407 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:21,407 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:21,578 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/B/0556b5f672434a4eb85281e93ffa0c4f 2024-12-07T15:16:21,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/C/77f30e1d9c8b4bd28fbe1bd895182358 is 50, key is test_row_0/C:col10/1733584578590/Put/seqid=0 2024-12-07T15:16:21,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742416_1592 (size=12151) 2024-12-07T15:16:21,588 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/C/77f30e1d9c8b4bd28fbe1bd895182358 2024-12-07T15:16:21,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/A/bec8e521b49640b8b6249b6755c11ce8 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/bec8e521b49640b8b6249b6755c11ce8 2024-12-07T15:16:21,597 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/bec8e521b49640b8b6249b6755c11ce8, entries=150, sequenceid=206, filesize=30.4 K 2024-12-07T15:16:21,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/B/0556b5f672434a4eb85281e93ffa0c4f as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/0556b5f672434a4eb85281e93ffa0c4f 2024-12-07T15:16:21,601 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/0556b5f672434a4eb85281e93ffa0c4f, entries=150, sequenceid=206, filesize=11.9 K 2024-12-07T15:16:21,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/C/77f30e1d9c8b4bd28fbe1bd895182358 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/77f30e1d9c8b4bd28fbe1bd895182358 2024-12-07T15:16:21,605 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/77f30e1d9c8b4bd28fbe1bd895182358, entries=150, sequenceid=206, filesize=11.9 K 2024-12-07T15:16:21,606 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 069dbda53fbb065f68f88519d6f94bc4 in 1257ms, sequenceid=206, compaction requested=false 2024-12-07T15:16:21,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2603): Flush status journal for 069dbda53fbb065f68f88519d6f94bc4: 2024-12-07T15:16:21,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:21,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=167 2024-12-07T15:16:21,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4169): Remote procedure done, pid=167 2024-12-07T15:16:21,610 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-12-07T15:16:21,610 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4120 sec 2024-12-07T15:16:21,613 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees in 1.4160 sec 2024-12-07T15:16:21,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on 069dbda53fbb065f68f88519d6f94bc4 2024-12-07T15:16:21,916 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 069dbda53fbb065f68f88519d6f94bc4 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-07T15:16:21,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 069dbda53fbb065f68f88519d6f94bc4, store=A 2024-12-07T15:16:21,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:16:21,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 069dbda53fbb065f68f88519d6f94bc4, store=B 2024-12-07T15:16:21,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:16:21,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 069dbda53fbb065f68f88519d6f94bc4, store=C 2024-12-07T15:16:21,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:16:21,923 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412077a8ea6ec6fb048bf8c45652cb54a2ea3_069dbda53fbb065f68f88519d6f94bc4 is 50, key is test_row_0/A:col10/1733584580770/Put/seqid=0 2024-12-07T15:16:21,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742417_1593 (size=12304) 2024-12-07T15:16:21,933 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:16:21,936 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412077a8ea6ec6fb048bf8c45652cb54a2ea3_069dbda53fbb065f68f88519d6f94bc4 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412077a8ea6ec6fb048bf8c45652cb54a2ea3_069dbda53fbb065f68f88519d6f94bc4 2024-12-07T15:16:21,937 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/A/5ea8ad4d6d414eefaa632767ce2dcc19, store: [table=TestAcidGuarantees family=A region=069dbda53fbb065f68f88519d6f94bc4] 2024-12-07T15:16:21,938 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/A/5ea8ad4d6d414eefaa632767ce2dcc19 is 175, key is test_row_0/A:col10/1733584580770/Put/seqid=0 2024-12-07T15:16:21,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742418_1594 (size=31105) 2024-12-07T15:16:21,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:21,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44694 deadline: 1733584641940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:21,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:21,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44662 deadline: 1733584641940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:21,945 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:21,945 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:21,945 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:21,945 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:21,945 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:21,945 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:22,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:22,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44694 deadline: 1733584642053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:22,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:22,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44662 deadline: 1733584642053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:22,057 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:22,057 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:22,057 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:22,057 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:22,057 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:22,057 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:22,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:22,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44694 deadline: 1733584642263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:22,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:22,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44662 deadline: 1733584642263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:22,267 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:22,267 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:22,267 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:22,267 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:22,267 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:22,267 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:22,343 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=225, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/A/5ea8ad4d6d414eefaa632767ce2dcc19 2024-12-07T15:16:22,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-07T15:16:22,343 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T15:16:22,344 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T15:16:22,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees 2024-12-07T15:16:22,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-07T15:16:22,346 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T15:16:22,347 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T15:16:22,347 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T15:16:22,350 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/B/c714460e3a8d4fc5bd2752f3d351bba6 is 50, key is test_row_0/B:col10/1733584580770/Put/seqid=0 2024-12-07T15:16:22,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742419_1595 (size=12151) 2024-12-07T15:16:22,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-07T15:16:22,499 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-12-07T15:16:22,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:22,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. as already flushing 2024-12-07T15:16:22,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:22,499 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:16:22,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:16:22,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:16:22,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:22,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44694 deadline: 1733584642573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:22,576 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:22,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:22,576 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:22,576 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:22,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44662 deadline: 1733584642573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:22,576 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:22,576 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:22,576 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:22,652 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-12-07T15:16:22,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:22,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. as already flushing 2024-12-07T15:16:22,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:22,652 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:16:22,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:16:22,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:16:22,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-07T15:16:22,753 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/B/c714460e3a8d4fc5bd2752f3d351bba6 2024-12-07T15:16:22,759 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/C/f10a3461d64f41b28fce5a1411358208 is 50, key is test_row_0/C:col10/1733584580770/Put/seqid=0 2024-12-07T15:16:22,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742420_1596 (size=12151) 2024-12-07T15:16:22,763 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/C/f10a3461d64f41b28fce5a1411358208 2024-12-07T15:16:22,766 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/A/5ea8ad4d6d414eefaa632767ce2dcc19 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/5ea8ad4d6d414eefaa632767ce2dcc19 2024-12-07T15:16:22,771 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/5ea8ad4d6d414eefaa632767ce2dcc19, entries=150, sequenceid=225, filesize=30.4 K 2024-12-07T15:16:22,771 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/B/c714460e3a8d4fc5bd2752f3d351bba6 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/c714460e3a8d4fc5bd2752f3d351bba6 2024-12-07T15:16:22,774 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/c714460e3a8d4fc5bd2752f3d351bba6, entries=150, sequenceid=225, filesize=11.9 K 2024-12-07T15:16:22,774 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/C/f10a3461d64f41b28fce5a1411358208 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/f10a3461d64f41b28fce5a1411358208 2024-12-07T15:16:22,777 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/f10a3461d64f41b28fce5a1411358208, entries=150, sequenceid=225, filesize=11.9 K 2024-12-07T15:16:22,778 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 069dbda53fbb065f68f88519d6f94bc4 in 862ms, sequenceid=225, compaction requested=true 2024-12-07T15:16:22,778 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 069dbda53fbb065f68f88519d6f94bc4: 2024-12-07T15:16:22,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 069dbda53fbb065f68f88519d6f94bc4:A, priority=-2147483648, current under compaction store size is 1 2024-12-07T15:16:22,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:16:22,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 069dbda53fbb065f68f88519d6f94bc4:B, priority=-2147483648, current under compaction store size is 2 2024-12-07T15:16:22,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:16:22,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 069dbda53fbb065f68f88519d6f94bc4:C, priority=-2147483648, current under compaction store size is 3 2024-12-07T15:16:22,778 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:16:22,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T15:16:22,778 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:16:22,779 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93725 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:16:22,779 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:16:22,779 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1541): 069dbda53fbb065f68f88519d6f94bc4/A is initiating minor compaction (all files) 2024-12-07T15:16:22,779 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1541): 069dbda53fbb065f68f88519d6f94bc4/B is initiating minor compaction (all files) 2024-12-07T15:16:22,779 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 069dbda53fbb065f68f88519d6f94bc4/A in TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:22,779 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 069dbda53fbb065f68f88519d6f94bc4/B in TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:22,779 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/e04d234b06fd429d89a2a7c6d92f0e84, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/bec8e521b49640b8b6249b6755c11ce8, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/5ea8ad4d6d414eefaa632767ce2dcc19] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp, totalSize=91.5 K 2024-12-07T15:16:22,779 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/f18c129635714bac831c83be303cac5c, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/0556b5f672434a4eb85281e93ffa0c4f, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/c714460e3a8d4fc5bd2752f3d351bba6] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp, totalSize=36.0 K 2024-12-07T15:16:22,779 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:22,779 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. files: [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/e04d234b06fd429d89a2a7c6d92f0e84, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/bec8e521b49640b8b6249b6755c11ce8, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/5ea8ad4d6d414eefaa632767ce2dcc19] 2024-12-07T15:16:22,779 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting f18c129635714bac831c83be303cac5c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1733584577930 2024-12-07T15:16:22,779 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting e04d234b06fd429d89a2a7c6d92f0e84, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1733584577930 2024-12-07T15:16:22,780 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 0556b5f672434a4eb85281e93ffa0c4f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1733584578587 2024-12-07T15:16:22,780 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting bec8e521b49640b8b6249b6755c11ce8, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1733584578587 2024-12-07T15:16:22,780 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting c714460e3a8d4fc5bd2752f3d351bba6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1733584580762 2024-12-07T15:16:22,780 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ea8ad4d6d414eefaa632767ce2dcc19, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1733584580762 2024-12-07T15:16:22,785 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=069dbda53fbb065f68f88519d6f94bc4] 2024-12-07T15:16:22,788 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 069dbda53fbb065f68f88519d6f94bc4#B#compaction#506 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:16:22,788 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/B/fc5f9d4d8b2d4ae3a50b12ce550a2dfe is 50, key is test_row_0/B:col10/1733584580770/Put/seqid=0 2024-12-07T15:16:22,789 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120780934774d44f4e619e91da866f6ab874_069dbda53fbb065f68f88519d6f94bc4 store=[table=TestAcidGuarantees family=A region=069dbda53fbb065f68f88519d6f94bc4] 2024-12-07T15:16:22,791 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120780934774d44f4e619e91da866f6ab874_069dbda53fbb065f68f88519d6f94bc4, store=[table=TestAcidGuarantees family=A region=069dbda53fbb065f68f88519d6f94bc4] 2024-12-07T15:16:22,791 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120780934774d44f4e619e91da866f6ab874_069dbda53fbb065f68f88519d6f94bc4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=069dbda53fbb065f68f88519d6f94bc4] 2024-12-07T15:16:22,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742421_1597 (size=12663) 2024-12-07T15:16:22,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742422_1598 (size=4469) 2024-12-07T15:16:22,797 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 069dbda53fbb065f68f88519d6f94bc4#A#compaction#505 average throughput is 2.04 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:16:22,798 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/A/dd9a45e1707540ff84152f2de4532403 is 175, key is test_row_0/A:col10/1733584580770/Put/seqid=0 2024-12-07T15:16:22,799 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/B/fc5f9d4d8b2d4ae3a50b12ce550a2dfe as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/fc5f9d4d8b2d4ae3a50b12ce550a2dfe 2024-12-07T15:16:22,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742423_1599 (size=31617) 2024-12-07T15:16:22,804 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 069dbda53fbb065f68f88519d6f94bc4/B of 069dbda53fbb065f68f88519d6f94bc4 into fc5f9d4d8b2d4ae3a50b12ce550a2dfe(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:16:22,804 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 069dbda53fbb065f68f88519d6f94bc4: 2024-12-07T15:16:22,804 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., storeName=069dbda53fbb065f68f88519d6f94bc4/B, priority=13, startTime=1733584582778; duration=0sec 2024-12-07T15:16:22,804 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T15:16:22,804 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 069dbda53fbb065f68f88519d6f94bc4:B 2024-12-07T15:16:22,804 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:16:22,804 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-12-07T15:16:22,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:22,805 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2902): Flushing 069dbda53fbb065f68f88519d6f94bc4 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-07T15:16:22,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 069dbda53fbb065f68f88519d6f94bc4, store=A 2024-12-07T15:16:22,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:16:22,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 069dbda53fbb065f68f88519d6f94bc4, store=B 2024-12-07T15:16:22,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:16:22,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 069dbda53fbb065f68f88519d6f94bc4, store=C 2024-12-07T15:16:22,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:16:22,806 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:16:22,806 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1541): 069dbda53fbb065f68f88519d6f94bc4/C is initiating minor compaction (all files) 2024-12-07T15:16:22,806 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 069dbda53fbb065f68f88519d6f94bc4/C in TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:22,806 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/5a91b0641da04d5b9a1ee6718f29139f, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/77f30e1d9c8b4bd28fbe1bd895182358, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/f10a3461d64f41b28fce5a1411358208] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp, totalSize=36.0 K 2024-12-07T15:16:22,806 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a91b0641da04d5b9a1ee6718f29139f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1733584577930 2024-12-07T15:16:22,807 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 77f30e1d9c8b4bd28fbe1bd895182358, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1733584578587 2024-12-07T15:16:22,808 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting f10a3461d64f41b28fce5a1411358208, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1733584580762 2024-12-07T15:16:22,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412077ffb7d497a244a8f8cdf06952994f5fd_069dbda53fbb065f68f88519d6f94bc4 is 50, key is test_row_0/A:col10/1733584581931/Put/seqid=0 2024-12-07T15:16:22,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742424_1600 (size=12304) 2024-12-07T15:16:22,821 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 069dbda53fbb065f68f88519d6f94bc4#C#compaction#508 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:16:22,821 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/C/b4cee46608704c61a33f2a7ae737111b is 50, key is test_row_0/C:col10/1733584580770/Put/seqid=0 2024-12-07T15:16:22,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742425_1601 (size=12663) 2024-12-07T15:16:22,830 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/C/b4cee46608704c61a33f2a7ae737111b as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/b4cee46608704c61a33f2a7ae737111b 2024-12-07T15:16:22,836 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 069dbda53fbb065f68f88519d6f94bc4/C of 069dbda53fbb065f68f88519d6f94bc4 into b4cee46608704c61a33f2a7ae737111b(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:16:22,836 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 069dbda53fbb065f68f88519d6f94bc4: 2024-12-07T15:16:22,836 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., storeName=069dbda53fbb065f68f88519d6f94bc4/C, priority=13, startTime=1733584582778; duration=0sec 2024-12-07T15:16:22,836 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:16:22,836 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 069dbda53fbb065f68f88519d6f94bc4:C 2024-12-07T15:16:22,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-07T15:16:23,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on 069dbda53fbb065f68f88519d6f94bc4 2024-12-07T15:16:23,084 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. as already flushing 2024-12-07T15:16:23,129 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:23,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44694 deadline: 1733584643123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:23,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:23,130 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:23,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44662 deadline: 1733584643124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:23,130 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:23,130 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:23,130 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:23,130 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:23,130 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:23,206 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/A/dd9a45e1707540ff84152f2de4532403 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/dd9a45e1707540ff84152f2de4532403 2024-12-07T15:16:23,212 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 069dbda53fbb065f68f88519d6f94bc4/A of 069dbda53fbb065f68f88519d6f94bc4 into dd9a45e1707540ff84152f2de4532403(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:16:23,212 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 069dbda53fbb065f68f88519d6f94bc4: 2024-12-07T15:16:23,212 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., storeName=069dbda53fbb065f68f88519d6f94bc4/A, priority=13, startTime=1733584582778; duration=0sec 2024-12-07T15:16:23,212 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:16:23,212 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 069dbda53fbb065f68f88519d6f94bc4:A 2024-12-07T15:16:23,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:16:23,228 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412077ffb7d497a244a8f8cdf06952994f5fd_069dbda53fbb065f68f88519d6f94bc4 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412077ffb7d497a244a8f8cdf06952994f5fd_069dbda53fbb065f68f88519d6f94bc4 2024-12-07T15:16:23,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/A/def128dc718e4ca6aa41451e940fe10a, store: [table=TestAcidGuarantees family=A region=069dbda53fbb065f68f88519d6f94bc4] 2024-12-07T15:16:23,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/A/def128dc718e4ca6aa41451e940fe10a is 175, key is test_row_0/A:col10/1733584581931/Put/seqid=0 2024-12-07T15:16:23,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742426_1602 (size=31105) 2024-12-07T15:16:23,238 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:23,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44694 deadline: 1733584643233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:23,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:23,239 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:23,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44662 deadline: 1733584643233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:23,239 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:23,239 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:23,239 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:23,239 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:23,239 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:23,240 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=239, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/A/def128dc718e4ca6aa41451e940fe10a 2024-12-07T15:16:23,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/B/797e212393d14ceea3221b17019560a8 is 50, key is test_row_0/B:col10/1733584581931/Put/seqid=0 2024-12-07T15:16:23,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742427_1603 (size=12151) 2024-12-07T15:16:23,262 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/B/797e212393d14ceea3221b17019560a8 2024-12-07T15:16:23,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/C/363b4741cddf4ef18d59f14eec252152 is 50, key is test_row_0/C:col10/1733584581931/Put/seqid=0 2024-12-07T15:16:23,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742428_1604 (size=12151) 2024-12-07T15:16:23,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:23,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44694 deadline: 1733584643443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:23,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:23,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44662 deadline: 1733584643443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:23,445 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:23,445 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:23,445 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:23,445 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:23,445 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:23,445 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:23,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-07T15:16:23,672 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/C/363b4741cddf4ef18d59f14eec252152 2024-12-07T15:16:23,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/A/def128dc718e4ca6aa41451e940fe10a as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/def128dc718e4ca6aa41451e940fe10a 2024-12-07T15:16:23,678 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/def128dc718e4ca6aa41451e940fe10a, entries=150, sequenceid=239, filesize=30.4 K 2024-12-07T15:16:23,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/B/797e212393d14ceea3221b17019560a8 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/797e212393d14ceea3221b17019560a8 2024-12-07T15:16:23,682 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/797e212393d14ceea3221b17019560a8, entries=150, sequenceid=239, filesize=11.9 K 2024-12-07T15:16:23,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/C/363b4741cddf4ef18d59f14eec252152 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/363b4741cddf4ef18d59f14eec252152 2024-12-07T15:16:23,685 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/363b4741cddf4ef18d59f14eec252152, entries=150, sequenceid=239, filesize=11.9 K 2024-12-07T15:16:23,686 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 069dbda53fbb065f68f88519d6f94bc4 in 881ms, sequenceid=239, compaction requested=false 2024-12-07T15:16:23,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2603): Flush status journal for 069dbda53fbb065f68f88519d6f94bc4: 2024-12-07T15:16:23,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:23,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=169 2024-12-07T15:16:23,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4169): Remote procedure done, pid=169 2024-12-07T15:16:23,689 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=169, resume processing ppid=168 2024-12-07T15:16:23,689 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, ppid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3400 sec 2024-12-07T15:16:23,690 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees in 1.3450 sec 2024-12-07T15:16:23,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on 069dbda53fbb065f68f88519d6f94bc4 2024-12-07T15:16:23,760 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 069dbda53fbb065f68f88519d6f94bc4 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-07T15:16:23,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 069dbda53fbb065f68f88519d6f94bc4, store=A 2024-12-07T15:16:23,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:16:23,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 069dbda53fbb065f68f88519d6f94bc4, store=B 2024-12-07T15:16:23,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:16:23,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 069dbda53fbb065f68f88519d6f94bc4, store=C 2024-12-07T15:16:23,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:16:23,768 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207299a0d5d243746d9ba3102b63368965f_069dbda53fbb065f68f88519d6f94bc4 is 50, key is test_row_0/A:col10/1733584583759/Put/seqid=0 2024-12-07T15:16:23,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742429_1605 (size=12454) 2024-12-07T15:16:23,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:23,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44694 deadline: 1733584643778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:23,782 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:23,782 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:23,782 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:23,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:23,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44662 deadline: 1733584643781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:23,785 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:23,785 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:23,785 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:23,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:23,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44650 deadline: 1733584643893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:23,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:23,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44662 deadline: 1733584643894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:23,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:23,897 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:23,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:23,897 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:23,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44694 deadline: 1733584643894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:23,897 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:23,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44682 deadline: 1733584643894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:23,897 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:23,897 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to d6797ebe9c52,46009,1733584418401 for 'test_row_1' in 069dbda53fbb065f68f88519d6f94bc4 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18248 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-07T15:16:23,898 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:23,898 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:23,898 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:23,898 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:23,898 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:23,898 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to d6797ebe9c52,46009,1733584418401 for 'test_row_2' in 069dbda53fbb065f68f88519d6f94bc4 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18248 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-07T15:16:23,898 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:23,898 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:23,898 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:23,915 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:23,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44654 deadline: 1733584643913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:23,916 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to d6797ebe9c52,46009,1733584418401 for 'test_row_0' in 069dbda53fbb065f68f88519d6f94bc4 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18266 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-07T15:16:23,916 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:23,916 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:23,916 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:24,106 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:24,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44662 deadline: 1733584644103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:24,106 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:24,106 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:24,107 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:24,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44694 deadline: 1733584644104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:24,107 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:24,107 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:24,107 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:24,107 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:24,171 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:16:24,175 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207299a0d5d243746d9ba3102b63368965f_069dbda53fbb065f68f88519d6f94bc4 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207299a0d5d243746d9ba3102b63368965f_069dbda53fbb065f68f88519d6f94bc4 2024-12-07T15:16:24,175 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/A/71f5c772ae1b45c7812658592c864bf6, store: [table=TestAcidGuarantees family=A region=069dbda53fbb065f68f88519d6f94bc4] 2024-12-07T15:16:24,176 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/A/71f5c772ae1b45c7812658592c864bf6 is 175, key is test_row_0/A:col10/1733584583759/Put/seqid=0 2024-12-07T15:16:24,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742430_1606 (size=31255) 2024-12-07T15:16:24,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:24,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44662 deadline: 1733584644413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:24,417 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:24,417 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:24,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44694 deadline: 1733584644413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:24,417 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:24,417 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:24,417 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:24,417 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:24,417 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:24,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-07T15:16:24,503 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T15:16:24,504 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T15:16:24,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] procedure2.ProcedureExecutor(1139): Stored pid=170, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees 2024-12-07T15:16:24,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-12-07T15:16:24,506 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=170, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T15:16:24,506 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=170, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T15:16:24,506 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=171, ppid=170, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T15:16:24,581 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=261, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/A/71f5c772ae1b45c7812658592c864bf6 2024-12-07T15:16:24,587 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/B/e249c831dabe42e3a5a109b3e2740845 is 50, key is test_row_0/B:col10/1733584583759/Put/seqid=0 2024-12-07T15:16:24,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742431_1607 (size=12301) 2024-12-07T15:16:24,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-12-07T15:16:24,659 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-12-07T15:16:24,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:24,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. as already flushing 2024-12-07T15:16:24,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:24,659 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:16:24,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:16:24,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:16:24,812 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-12-07T15:16:24,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:24,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. as already flushing 2024-12-07T15:16:24,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:24,812 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:16:24,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:16:24,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:16:24,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-12-07T15:16:24,925 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:24,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44662 deadline: 1733584644923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:24,926 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T15:16:24,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46009 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44694 deadline: 1733584644923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:24,926 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:24,926 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:24,926 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:24,926 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:24,926 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T15:16:24,926 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., hostname=d6797ebe9c52,46009,1733584418401, seqNum=5 because the exception is null or not the one we care about 2024-12-07T15:16:24,965 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-12-07T15:16:24,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:24,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. as already flushing 2024-12-07T15:16:24,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:24,965 ERROR [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:16:24,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:16:24,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T15:16:24,991 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/B/e249c831dabe42e3a5a109b3e2740845 2024-12-07T15:16:24,997 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/C/924683ad4a964bc0a951353c336b6672 is 50, key is test_row_0/C:col10/1733584583759/Put/seqid=0 2024-12-07T15:16:25,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742432_1608 (size=12301) 2024-12-07T15:16:25,001 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/C/924683ad4a964bc0a951353c336b6672 2024-12-07T15:16:25,004 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/A/71f5c772ae1b45c7812658592c864bf6 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/71f5c772ae1b45c7812658592c864bf6 2024-12-07T15:16:25,007 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/71f5c772ae1b45c7812658592c864bf6, entries=150, sequenceid=261, filesize=30.5 K 2024-12-07T15:16:25,008 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/B/e249c831dabe42e3a5a109b3e2740845 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/e249c831dabe42e3a5a109b3e2740845 2024-12-07T15:16:25,011 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/e249c831dabe42e3a5a109b3e2740845, entries=150, sequenceid=261, filesize=12.0 K 2024-12-07T15:16:25,012 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/C/924683ad4a964bc0a951353c336b6672 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/924683ad4a964bc0a951353c336b6672 2024-12-07T15:16:25,015 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/924683ad4a964bc0a951353c336b6672, entries=150, sequenceid=261, filesize=12.0 K 2024-12-07T15:16:25,015 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 069dbda53fbb065f68f88519d6f94bc4 in 1255ms, sequenceid=261, compaction requested=true 2024-12-07T15:16:25,015 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 069dbda53fbb065f68f88519d6f94bc4: 2024-12-07T15:16:25,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 069dbda53fbb065f68f88519d6f94bc4:A, priority=-2147483648, current under compaction store size is 1 2024-12-07T15:16:25,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:16:25,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 069dbda53fbb065f68f88519d6f94bc4:B, priority=-2147483648, current under compaction store size is 2 2024-12-07T15:16:25,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:16:25,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 069dbda53fbb065f68f88519d6f94bc4:C, priority=-2147483648, current under compaction store size is 3 2024-12-07T15:16:25,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T15:16:25,016 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:16:25,016 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:16:25,016 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:16:25,016 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93977 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:16:25,016 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1541): 069dbda53fbb065f68f88519d6f94bc4/B is initiating minor compaction (all files) 2024-12-07T15:16:25,016 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1541): 069dbda53fbb065f68f88519d6f94bc4/A is initiating minor compaction (all files) 2024-12-07T15:16:25,017 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 069dbda53fbb065f68f88519d6f94bc4/B in TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:25,017 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 069dbda53fbb065f68f88519d6f94bc4/A in TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:25,017 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/fc5f9d4d8b2d4ae3a50b12ce550a2dfe, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/797e212393d14ceea3221b17019560a8, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/e249c831dabe42e3a5a109b3e2740845] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp, totalSize=36.2 K 2024-12-07T15:16:25,017 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/dd9a45e1707540ff84152f2de4532403, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/def128dc718e4ca6aa41451e940fe10a, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/71f5c772ae1b45c7812658592c864bf6] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp, totalSize=91.8 K 2024-12-07T15:16:25,017 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:25,017 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. files: [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/dd9a45e1707540ff84152f2de4532403, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/def128dc718e4ca6aa41451e940fe10a, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/71f5c772ae1b45c7812658592c864bf6] 2024-12-07T15:16:25,017 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting fc5f9d4d8b2d4ae3a50b12ce550a2dfe, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1733584580762 2024-12-07T15:16:25,017 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd9a45e1707540ff84152f2de4532403, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1733584580762 2024-12-07T15:16:25,017 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 797e212393d14ceea3221b17019560a8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1733584581931 2024-12-07T15:16:25,018 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting def128dc718e4ca6aa41451e940fe10a, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1733584581931 2024-12-07T15:16:25,018 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting e249c831dabe42e3a5a109b3e2740845, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1733584583121 2024-12-07T15:16:25,018 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] compactions.Compactor(225): Compacting 71f5c772ae1b45c7812658592c864bf6, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1733584583121 2024-12-07T15:16:25,024 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=069dbda53fbb065f68f88519d6f94bc4] 2024-12-07T15:16:25,025 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 069dbda53fbb065f68f88519d6f94bc4#B#compaction#514 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:16:25,025 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/B/fe0db4ce3b2642eaa13e191f3e3a2b95 is 50, key is test_row_0/B:col10/1733584583759/Put/seqid=0 2024-12-07T15:16:25,026 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241207554f0b4275f14a2aaea2088cf293ddbd_069dbda53fbb065f68f88519d6f94bc4 store=[table=TestAcidGuarantees family=A region=069dbda53fbb065f68f88519d6f94bc4] 2024-12-07T15:16:25,028 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241207554f0b4275f14a2aaea2088cf293ddbd_069dbda53fbb065f68f88519d6f94bc4, store=[table=TestAcidGuarantees family=A region=069dbda53fbb065f68f88519d6f94bc4] 2024-12-07T15:16:25,028 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207554f0b4275f14a2aaea2088cf293ddbd_069dbda53fbb065f68f88519d6f94bc4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=069dbda53fbb065f68f88519d6f94bc4] 2024-12-07T15:16:25,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742433_1609 (size=12915) 2024-12-07T15:16:25,035 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/B/fe0db4ce3b2642eaa13e191f3e3a2b95 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/fe0db4ce3b2642eaa13e191f3e3a2b95 2024-12-07T15:16:25,039 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 069dbda53fbb065f68f88519d6f94bc4/B of 069dbda53fbb065f68f88519d6f94bc4 into fe0db4ce3b2642eaa13e191f3e3a2b95(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:16:25,039 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 069dbda53fbb065f68f88519d6f94bc4: 2024-12-07T15:16:25,039 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., storeName=069dbda53fbb065f68f88519d6f94bc4/B, priority=13, startTime=1733584585016; duration=0sec 2024-12-07T15:16:25,039 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T15:16:25,039 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 069dbda53fbb065f68f88519d6f94bc4:B 2024-12-07T15:16:25,039 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T15:16:25,040 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T15:16:25,040 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1541): 069dbda53fbb065f68f88519d6f94bc4/C is initiating minor compaction (all files) 2024-12-07T15:16:25,040 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 069dbda53fbb065f68f88519d6f94bc4/C in TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:25,040 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/b4cee46608704c61a33f2a7ae737111b, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/363b4741cddf4ef18d59f14eec252152, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/924683ad4a964bc0a951353c336b6672] into tmpdir=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp, totalSize=36.2 K 2024-12-07T15:16:25,041 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting b4cee46608704c61a33f2a7ae737111b, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1733584580762 2024-12-07T15:16:25,041 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 363b4741cddf4ef18d59f14eec252152, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1733584581931 2024-12-07T15:16:25,041 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] compactions.Compactor(225): Compacting 924683ad4a964bc0a951353c336b6672, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1733584583121 2024-12-07T15:16:25,047 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 069dbda53fbb065f68f88519d6f94bc4#C#compaction#516 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:16:25,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742434_1610 (size=4469) 2024-12-07T15:16:25,047 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/C/d42ab26cc9544c64897e6f1b32904502 is 50, key is test_row_0/C:col10/1733584583759/Put/seqid=0 2024-12-07T15:16:25,048 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 069dbda53fbb065f68f88519d6f94bc4#A#compaction#515 average throughput is 1.02 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T15:16:25,049 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/A/0480f813b0f54c8a838c577f7ccbb7f7 is 175, key is test_row_0/A:col10/1733584583759/Put/seqid=0 2024-12-07T15:16:25,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742435_1611 (size=12915) 2024-12-07T15:16:25,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742436_1612 (size=31869) 2024-12-07T15:16:25,117 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46009 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-12-07T15:16:25,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:25,118 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2902): Flushing 069dbda53fbb065f68f88519d6f94bc4 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-07T15:16:25,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 069dbda53fbb065f68f88519d6f94bc4, store=A 2024-12-07T15:16:25,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:16:25,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 069dbda53fbb065f68f88519d6f94bc4, store=B 2024-12-07T15:16:25,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:16:25,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 069dbda53fbb065f68f88519d6f94bc4, store=C 2024-12-07T15:16:25,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:16:25,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412070f1e9b7d6a604ce8bfce9cbec5bc1630_069dbda53fbb065f68f88519d6f94bc4 is 50, key is test_row_0/A:col10/1733584583772/Put/seqid=0 2024-12-07T15:16:25,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742437_1613 (size=12454) 2024-12-07T15:16:25,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:16:25,133 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412070f1e9b7d6a604ce8bfce9cbec5bc1630_069dbda53fbb065f68f88519d6f94bc4 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412070f1e9b7d6a604ce8bfce9cbec5bc1630_069dbda53fbb065f68f88519d6f94bc4 2024-12-07T15:16:25,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-12-07T15:16:25,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/A/d7139f0d10bb4ee6aff55b614a67d795, store: [table=TestAcidGuarantees family=A region=069dbda53fbb065f68f88519d6f94bc4] 2024-12-07T15:16:25,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/A/d7139f0d10bb4ee6aff55b614a67d795 is 175, key is test_row_0/A:col10/1733584583772/Put/seqid=0 2024-12-07T15:16:25,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742438_1614 (size=31255) 2024-12-07T15:16:25,463 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/C/d42ab26cc9544c64897e6f1b32904502 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/d42ab26cc9544c64897e6f1b32904502 2024-12-07T15:16:25,465 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/A/0480f813b0f54c8a838c577f7ccbb7f7 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/0480f813b0f54c8a838c577f7ccbb7f7 2024-12-07T15:16:25,468 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 069dbda53fbb065f68f88519d6f94bc4/C of 069dbda53fbb065f68f88519d6f94bc4 into d42ab26cc9544c64897e6f1b32904502(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:16:25,468 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 069dbda53fbb065f68f88519d6f94bc4: 2024-12-07T15:16:25,468 INFO [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., storeName=069dbda53fbb065f68f88519d6f94bc4/C, priority=13, startTime=1733584585016; duration=0sec 2024-12-07T15:16:25,468 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:16:25,468 DEBUG [RS:0;d6797ebe9c52:46009-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 069dbda53fbb065f68f88519d6f94bc4:C 2024-12-07T15:16:25,469 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 069dbda53fbb065f68f88519d6f94bc4/A of 069dbda53fbb065f68f88519d6f94bc4 into 0480f813b0f54c8a838c577f7ccbb7f7(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T15:16:25,469 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 069dbda53fbb065f68f88519d6f94bc4: 2024-12-07T15:16:25,469 INFO [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., storeName=069dbda53fbb065f68f88519d6f94bc4/A, priority=13, startTime=1733584585015; duration=0sec 2024-12-07T15:16:25,469 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T15:16:25,469 DEBUG [RS:0;d6797ebe9c52:46009-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 069dbda53fbb065f68f88519d6f94bc4:A 2024-12-07T15:16:25,539 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=274, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/A/d7139f0d10bb4ee6aff55b614a67d795 2024-12-07T15:16:25,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/B/ae839e3e0fc2401f873851f7e5a4357a is 50, key is test_row_0/B:col10/1733584583772/Put/seqid=0 2024-12-07T15:16:25,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742439_1615 (size=12301) 2024-12-07T15:16:25,553 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/B/ae839e3e0fc2401f873851f7e5a4357a 2024-12-07T15:16:25,559 INFO [Thread-2400 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2400. 2024-12-07T15:16:25,559 DEBUG [Thread-2400 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T15:16:25,559 DEBUG [Thread-2400 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:16:25,559 DEBUG [Thread-2400 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:16:25,559 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T15:16:25,559 INFO [Thread-2404 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2404. 2024-12-07T15:16:25,559 DEBUG [Thread-2404 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T15:16:25,559 DEBUG [Thread-2404 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:16:25,560 DEBUG [Thread-2404 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:16:25,560 INFO [Thread-2406 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2406. 2024-12-07T15:16:25,560 INFO [Thread-2398 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2398. 2024-12-07T15:16:25,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/C/fee6cd142fb4482788cb317d213c1b02 is 50, key is test_row_0/C:col10/1733584583772/Put/seqid=0 2024-12-07T15:16:25,560 DEBUG [Thread-2398 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T15:16:25,560 DEBUG [Thread-2406 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T15:16:25,560 DEBUG [Thread-2398 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:16:25,560 DEBUG [Thread-2406 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:16:25,560 DEBUG [Thread-2398 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:16:25,560 DEBUG [Thread-2406 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:16:25,560 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T15:16:25,560 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T15:16:25,560 INFO [Thread-2402 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2402. 2024-12-07T15:16:25,560 DEBUG [Thread-2402 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T15:16:25,560 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T15:16:25,560 DEBUG [Thread-2402 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:16:25,560 DEBUG [Thread-2402 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:16:25,561 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T15:16:25,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742440_1616 (size=12301) 2024-12-07T15:16:25,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-12-07T15:16:25,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46009 {}] regionserver.HRegion(8855): Flush requested on 069dbda53fbb065f68f88519d6f94bc4 2024-12-07T15:16:25,935 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. as already flushing 2024-12-07T15:16:25,935 INFO [Thread-2393 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2393. 2024-12-07T15:16:25,935 DEBUG [Thread-2393 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T15:16:25,935 DEBUG [Thread-2393 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:16:25,935 DEBUG [Thread-2393 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:16:25,935 INFO [Thread-2389 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2389. 2024-12-07T15:16:25,935 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T15:16:25,935 DEBUG [Thread-2389 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T15:16:25,935 DEBUG [Thread-2389 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:16:25,935 DEBUG [Thread-2389 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:16:25,935 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T15:16:25,970 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/C/fee6cd142fb4482788cb317d213c1b02 2024-12-07T15:16:25,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/A/d7139f0d10bb4ee6aff55b614a67d795 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/d7139f0d10bb4ee6aff55b614a67d795 2024-12-07T15:16:25,976 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/d7139f0d10bb4ee6aff55b614a67d795, entries=150, sequenceid=274, filesize=30.5 K 2024-12-07T15:16:25,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/B/ae839e3e0fc2401f873851f7e5a4357a as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/ae839e3e0fc2401f873851f7e5a4357a 2024-12-07T15:16:25,979 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/ae839e3e0fc2401f873851f7e5a4357a, entries=150, sequenceid=274, filesize=12.0 K 2024-12-07T15:16:25,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/C/fee6cd142fb4482788cb317d213c1b02 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/fee6cd142fb4482788cb317d213c1b02 2024-12-07T15:16:25,982 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/fee6cd142fb4482788cb317d213c1b02, entries=150, sequenceid=274, filesize=12.0 K 2024-12-07T15:16:25,983 INFO [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=13.42 KB/13740 for 069dbda53fbb065f68f88519d6f94bc4 in 865ms, sequenceid=274, compaction requested=false 2024-12-07T15:16:25,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2603): Flush status journal for 069dbda53fbb065f68f88519d6f94bc4: 2024-12-07T15:16:25,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:25,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d6797ebe9c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=171 2024-12-07T15:16:25,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster(4169): Remote procedure done, pid=171 2024-12-07T15:16:25,985 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=171, resume processing ppid=170 2024-12-07T15:16:25,985 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=171, ppid=170, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4780 sec 2024-12-07T15:16:25,986 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees in 1.4820 sec 2024-12-07T15:16:26,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-12-07T15:16:26,654 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T15:16:33,915 INFO [Thread-2387 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2387. 2024-12-07T15:16:33,915 DEBUG [Thread-2387 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T15:16:33,915 DEBUG [Thread-2387 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:16:33,915 DEBUG [Thread-2387 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:16:33,915 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T15:16:33,984 INFO [Thread-2391 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2391. 2024-12-07T15:16:33,984 DEBUG [Thread-2391 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T15:16:33,984 DEBUG [Thread-2391 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:16:33,984 DEBUG [Thread-2391 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:16:33,984 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T15:16:34,004 INFO [Thread-2395 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2395. 2024-12-07T15:16:34,004 DEBUG [Thread-2395 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T15:16:34,004 DEBUG [Thread-2395 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:16:34,004 DEBUG [Thread-2395 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:16:34,004 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T15:16:34,004 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-07T15:16:34,004 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 11 2024-12-07T15:16:34,005 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 87 2024-12-07T15:16:34,005 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 12 2024-12-07T15:16:34,005 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 94 2024-12-07T15:16:34,005 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 13 2024-12-07T15:16:34,005 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-07T15:16:34,005 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3158 2024-12-07T15:16:34,005 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3083 2024-12-07T15:16:34,005 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3143 2024-12-07T15:16:34,005 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3142 2024-12-07T15:16:34,005 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3087 2024-12-07T15:16:34,005 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-07T15:16:34,005 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-07T15:16:34,005 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-07T15:16:34,005 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:16:34,005 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:16:34,005 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T15:16:34,006 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-07T15:16:34,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] procedure2.ProcedureExecutor(1139): Stored pid=172, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-07T15:16:34,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-12-07T15:16:34,008 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733584594008"}]},"ts":"1733584594008"} 2024-12-07T15:16:34,009 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-07T15:16:34,009 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-07T15:16:34,010 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=173, ppid=172, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-07T15:16:34,011 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=174, ppid=173, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=069dbda53fbb065f68f88519d6f94bc4, UNASSIGN}] 2024-12-07T15:16:34,011 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=174, ppid=173, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=069dbda53fbb065f68f88519d6f94bc4, UNASSIGN 2024-12-07T15:16:34,012 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=174 updating hbase:meta row=069dbda53fbb065f68f88519d6f94bc4, regionState=CLOSING, regionLocation=d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:34,013 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=174, ppid=173, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=069dbda53fbb065f68f88519d6f94bc4, UNASSIGN because future has completed 2024-12-07T15:16:34,013 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-07T15:16:34,013 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=175, ppid=174, state=RUNNABLE, hasLock=false; CloseRegionProcedure 069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401}] 2024-12-07T15:16:34,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-12-07T15:16:34,165 INFO [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] handler.UnassignRegionHandler(122): Close 069dbda53fbb065f68f88519d6f94bc4 2024-12-07T15:16:34,165 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-07T15:16:34,165 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1722): Closing 069dbda53fbb065f68f88519d6f94bc4, disabling compactions & flushes 2024-12-07T15:16:34,165 INFO [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:34,165 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:34,165 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. after waiting 0 ms 2024-12-07T15:16:34,165 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:34,165 INFO [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(2902): Flushing 069dbda53fbb065f68f88519d6f94bc4 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-07T15:16:34,165 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 069dbda53fbb065f68f88519d6f94bc4, store=A 2024-12-07T15:16:34,165 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:16:34,165 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 069dbda53fbb065f68f88519d6f94bc4, store=B 2024-12-07T15:16:34,165 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:16:34,165 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 069dbda53fbb065f68f88519d6f94bc4, store=C 2024-12-07T15:16:34,165 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T15:16:34,170 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207a247122e626b4b9b80b86e5efda28ddb_069dbda53fbb065f68f88519d6f94bc4 is 50, key is test_row_0/A:col10/1733584594003/Put/seqid=0 2024-12-07T15:16:34,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742441_1617 (size=12454) 2024-12-07T15:16:34,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-12-07T15:16:34,573 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T15:16:34,576 INFO [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207a247122e626b4b9b80b86e5efda28ddb_069dbda53fbb065f68f88519d6f94bc4 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207a247122e626b4b9b80b86e5efda28ddb_069dbda53fbb065f68f88519d6f94bc4 2024-12-07T15:16:34,577 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/A/e53766791bf04ad392e7f3cf311ed6ce, store: [table=TestAcidGuarantees family=A region=069dbda53fbb065f68f88519d6f94bc4] 2024-12-07T15:16:34,577 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/A/e53766791bf04ad392e7f3cf311ed6ce is 175, key is test_row_0/A:col10/1733584594003/Put/seqid=0 2024-12-07T15:16:34,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742442_1618 (size=31255) 2024-12-07T15:16:34,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-12-07T15:16:34,981 INFO [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=284, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/A/e53766791bf04ad392e7f3cf311ed6ce 2024-12-07T15:16:34,985 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/B/af5978079cd84775af33fce837da49e6 is 50, key is test_row_0/B:col10/1733584594003/Put/seqid=0 2024-12-07T15:16:34,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742443_1619 (size=12301) 2024-12-07T15:16:35,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-12-07T15:16:35,389 INFO [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/B/af5978079cd84775af33fce837da49e6 2024-12-07T15:16:35,394 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/C/446d4b95cd324914b62f6a3934fc3b0d is 50, key is test_row_0/C:col10/1733584594003/Put/seqid=0 2024-12-07T15:16:35,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742444_1620 (size=12301) 2024-12-07T15:16:35,797 INFO [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/C/446d4b95cd324914b62f6a3934fc3b0d 2024-12-07T15:16:35,800 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/A/e53766791bf04ad392e7f3cf311ed6ce as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/e53766791bf04ad392e7f3cf311ed6ce 2024-12-07T15:16:35,803 INFO [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/e53766791bf04ad392e7f3cf311ed6ce, entries=150, sequenceid=284, filesize=30.5 K 2024-12-07T15:16:35,803 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/B/af5978079cd84775af33fce837da49e6 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/af5978079cd84775af33fce837da49e6 2024-12-07T15:16:35,806 INFO [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/af5978079cd84775af33fce837da49e6, entries=150, sequenceid=284, filesize=12.0 K 2024-12-07T15:16:35,806 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/.tmp/C/446d4b95cd324914b62f6a3934fc3b0d as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/446d4b95cd324914b62f6a3934fc3b0d 2024-12-07T15:16:35,809 INFO [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/446d4b95cd324914b62f6a3934fc3b0d, entries=150, sequenceid=284, filesize=12.0 K 2024-12-07T15:16:35,809 INFO [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 069dbda53fbb065f68f88519d6f94bc4 in 1644ms, sequenceid=284, compaction requested=true 2024-12-07T15:16:35,810 DEBUG [StoreCloser-TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/8a7da40c81a14b9caf61636ea65f1e3c, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/807f4cbdc9724a3cbf0669d9ec00b6b0, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/aa2b61a68ad344a5b9247a5ddffb8dd9, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/d8819e606cf14e558a68af6cc2cc5299, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/df5054d67b1645de8116f9f6fa90796d, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/9555352250874630bd2b7ab3f9919451, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/ea5baf5c16e547129abe2a55b8b8e750, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/e33eeb0e0fcc42e79c7b593df3f57b7d, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/f3a899119ce84389843d56a6633adadf, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/c545dc54d2954d7883a655303693c480, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/b538aa39a4b04ab8a3e6dcf25a8f3567, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/e04d234b06fd429d89a2a7c6d92f0e84, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/a53bd14d01634c3d9932dddde412d364, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/bec8e521b49640b8b6249b6755c11ce8, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/dd9a45e1707540ff84152f2de4532403, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/5ea8ad4d6d414eefaa632767ce2dcc19, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/def128dc718e4ca6aa41451e940fe10a, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/71f5c772ae1b45c7812658592c864bf6] to archive 2024-12-07T15:16:35,810 DEBUG [StoreCloser-TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-07T15:16:35,811 DEBUG [StoreCloser-TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/8a7da40c81a14b9caf61636ea65f1e3c to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/8a7da40c81a14b9caf61636ea65f1e3c 2024-12-07T15:16:35,812 DEBUG [StoreCloser-TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/807f4cbdc9724a3cbf0669d9ec00b6b0 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/807f4cbdc9724a3cbf0669d9ec00b6b0 2024-12-07T15:16:35,813 DEBUG [StoreCloser-TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/aa2b61a68ad344a5b9247a5ddffb8dd9 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/aa2b61a68ad344a5b9247a5ddffb8dd9 2024-12-07T15:16:35,814 DEBUG [StoreCloser-TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/d8819e606cf14e558a68af6cc2cc5299 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/d8819e606cf14e558a68af6cc2cc5299 2024-12-07T15:16:35,814 DEBUG [StoreCloser-TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/df5054d67b1645de8116f9f6fa90796d to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/df5054d67b1645de8116f9f6fa90796d 2024-12-07T15:16:35,815 DEBUG [StoreCloser-TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/9555352250874630bd2b7ab3f9919451 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/9555352250874630bd2b7ab3f9919451 2024-12-07T15:16:35,816 DEBUG [StoreCloser-TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/ea5baf5c16e547129abe2a55b8b8e750 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/ea5baf5c16e547129abe2a55b8b8e750 2024-12-07T15:16:35,817 DEBUG [StoreCloser-TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/e33eeb0e0fcc42e79c7b593df3f57b7d to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/e33eeb0e0fcc42e79c7b593df3f57b7d 2024-12-07T15:16:35,818 DEBUG [StoreCloser-TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/f3a899119ce84389843d56a6633adadf to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/f3a899119ce84389843d56a6633adadf 2024-12-07T15:16:35,819 DEBUG [StoreCloser-TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/c545dc54d2954d7883a655303693c480 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/c545dc54d2954d7883a655303693c480 2024-12-07T15:16:35,819 DEBUG [StoreCloser-TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/b538aa39a4b04ab8a3e6dcf25a8f3567 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/b538aa39a4b04ab8a3e6dcf25a8f3567 2024-12-07T15:16:35,820 DEBUG [StoreCloser-TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/e04d234b06fd429d89a2a7c6d92f0e84 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/e04d234b06fd429d89a2a7c6d92f0e84 2024-12-07T15:16:35,821 DEBUG [StoreCloser-TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/a53bd14d01634c3d9932dddde412d364 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/a53bd14d01634c3d9932dddde412d364 2024-12-07T15:16:35,822 DEBUG [StoreCloser-TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/bec8e521b49640b8b6249b6755c11ce8 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/bec8e521b49640b8b6249b6755c11ce8 2024-12-07T15:16:35,823 DEBUG [StoreCloser-TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/dd9a45e1707540ff84152f2de4532403 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/dd9a45e1707540ff84152f2de4532403 2024-12-07T15:16:35,823 DEBUG [StoreCloser-TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/5ea8ad4d6d414eefaa632767ce2dcc19 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/5ea8ad4d6d414eefaa632767ce2dcc19 2024-12-07T15:16:35,824 DEBUG [StoreCloser-TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/def128dc718e4ca6aa41451e940fe10a to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/def128dc718e4ca6aa41451e940fe10a 2024-12-07T15:16:35,825 DEBUG [StoreCloser-TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/71f5c772ae1b45c7812658592c864bf6 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/71f5c772ae1b45c7812658592c864bf6 2024-12-07T15:16:35,826 DEBUG [StoreCloser-TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/e5d6283820c646069ecadd8849d6a487, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/ff9fab01304244688f1d8ee13883a5de, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/2a339d6a654e4b86940f554eae331e4e, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/579e1cd0111f47c98d488193543a2cb0, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/6147dbaaf78e4e03adaa546b103f1273, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/7efca4b8647a4590be0eba1441795fb4, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/8e11b48d6efe448e966c83661a5579d2, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/0d94443d2dc04c498f00a1894a810788, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/637f2b0698284c5fbb5a1e20fd6ac301, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/94b3d8797c8b4878acc10d50f2617898, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/2f2bb04f49dc48719ae99e340402a33a, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/f18c129635714bac831c83be303cac5c, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/c9c7be4bf0644dceb6e332780e8315ee, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/0556b5f672434a4eb85281e93ffa0c4f, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/fc5f9d4d8b2d4ae3a50b12ce550a2dfe, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/c714460e3a8d4fc5bd2752f3d351bba6, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/797e212393d14ceea3221b17019560a8, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/e249c831dabe42e3a5a109b3e2740845] to archive 2024-12-07T15:16:35,827 DEBUG [StoreCloser-TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-07T15:16:35,827 DEBUG [StoreCloser-TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/e5d6283820c646069ecadd8849d6a487 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/e5d6283820c646069ecadd8849d6a487 2024-12-07T15:16:35,828 DEBUG [StoreCloser-TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/ff9fab01304244688f1d8ee13883a5de to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/ff9fab01304244688f1d8ee13883a5de 2024-12-07T15:16:35,829 DEBUG [StoreCloser-TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/2a339d6a654e4b86940f554eae331e4e to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/2a339d6a654e4b86940f554eae331e4e 2024-12-07T15:16:35,830 DEBUG [StoreCloser-TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/579e1cd0111f47c98d488193543a2cb0 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/579e1cd0111f47c98d488193543a2cb0 2024-12-07T15:16:35,830 DEBUG [StoreCloser-TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/6147dbaaf78e4e03adaa546b103f1273 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/6147dbaaf78e4e03adaa546b103f1273 2024-12-07T15:16:35,831 DEBUG [StoreCloser-TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/7efca4b8647a4590be0eba1441795fb4 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/7efca4b8647a4590be0eba1441795fb4 2024-12-07T15:16:35,832 DEBUG [StoreCloser-TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/8e11b48d6efe448e966c83661a5579d2 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/8e11b48d6efe448e966c83661a5579d2 2024-12-07T15:16:35,833 DEBUG [StoreCloser-TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/0d94443d2dc04c498f00a1894a810788 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/0d94443d2dc04c498f00a1894a810788 2024-12-07T15:16:35,833 DEBUG [StoreCloser-TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/637f2b0698284c5fbb5a1e20fd6ac301 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/637f2b0698284c5fbb5a1e20fd6ac301 2024-12-07T15:16:35,834 DEBUG [StoreCloser-TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/94b3d8797c8b4878acc10d50f2617898 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/94b3d8797c8b4878acc10d50f2617898 2024-12-07T15:16:35,835 DEBUG [StoreCloser-TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/2f2bb04f49dc48719ae99e340402a33a to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/2f2bb04f49dc48719ae99e340402a33a 2024-12-07T15:16:35,836 DEBUG [StoreCloser-TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/f18c129635714bac831c83be303cac5c to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/f18c129635714bac831c83be303cac5c 2024-12-07T15:16:35,836 DEBUG [StoreCloser-TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/c9c7be4bf0644dceb6e332780e8315ee to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/c9c7be4bf0644dceb6e332780e8315ee 2024-12-07T15:16:35,837 DEBUG [StoreCloser-TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/0556b5f672434a4eb85281e93ffa0c4f to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/0556b5f672434a4eb85281e93ffa0c4f 2024-12-07T15:16:35,838 DEBUG [StoreCloser-TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/fc5f9d4d8b2d4ae3a50b12ce550a2dfe to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/fc5f9d4d8b2d4ae3a50b12ce550a2dfe 2024-12-07T15:16:35,839 DEBUG [StoreCloser-TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/c714460e3a8d4fc5bd2752f3d351bba6 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/c714460e3a8d4fc5bd2752f3d351bba6 2024-12-07T15:16:35,839 DEBUG [StoreCloser-TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/797e212393d14ceea3221b17019560a8 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/797e212393d14ceea3221b17019560a8 2024-12-07T15:16:35,840 DEBUG [StoreCloser-TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/e249c831dabe42e3a5a109b3e2740845 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/e249c831dabe42e3a5a109b3e2740845 2024-12-07T15:16:35,841 DEBUG [StoreCloser-TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/7b7e83ef764645b59235d8f94087bc56, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/fc8c540115a04e5680fcce693e890ac9, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/bf759a03a63644ce8bbeace14a4c8b58, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/3c9d1ff9ee794047b33ae2fd9df73b05, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/3c99821889c24a41b4c0e1662092eec9, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/e5355ae86dcb4e559c62e86a432236ea, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/9dbe03f66c824444bed9465c040bb33c, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/f3235915c4e94cb2b4eb1a98af2dfd87, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/f0f9bf5ef8f84566a416549159c55b3f, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/8360848629f44b259dedc3fa0e8b9183, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/8125d0e876ae445390a4cbf68416d7fc, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/5a91b0641da04d5b9a1ee6718f29139f, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/c5ccf9755f644ac29d9cf2e061d5993a, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/77f30e1d9c8b4bd28fbe1bd895182358, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/b4cee46608704c61a33f2a7ae737111b, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/f10a3461d64f41b28fce5a1411358208, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/363b4741cddf4ef18d59f14eec252152, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/924683ad4a964bc0a951353c336b6672] to archive 2024-12-07T15:16:35,842 DEBUG [StoreCloser-TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-07T15:16:35,843 DEBUG [StoreCloser-TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/7b7e83ef764645b59235d8f94087bc56 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/7b7e83ef764645b59235d8f94087bc56 2024-12-07T15:16:35,844 DEBUG [StoreCloser-TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/fc8c540115a04e5680fcce693e890ac9 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/fc8c540115a04e5680fcce693e890ac9 2024-12-07T15:16:35,844 DEBUG [StoreCloser-TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/bf759a03a63644ce8bbeace14a4c8b58 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/bf759a03a63644ce8bbeace14a4c8b58 2024-12-07T15:16:35,845 DEBUG [StoreCloser-TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/3c9d1ff9ee794047b33ae2fd9df73b05 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/3c9d1ff9ee794047b33ae2fd9df73b05 2024-12-07T15:16:35,846 DEBUG [StoreCloser-TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/3c99821889c24a41b4c0e1662092eec9 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/3c99821889c24a41b4c0e1662092eec9 2024-12-07T15:16:35,847 DEBUG [StoreCloser-TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/e5355ae86dcb4e559c62e86a432236ea to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/e5355ae86dcb4e559c62e86a432236ea 2024-12-07T15:16:35,848 DEBUG [StoreCloser-TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/9dbe03f66c824444bed9465c040bb33c to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/9dbe03f66c824444bed9465c040bb33c 2024-12-07T15:16:35,849 DEBUG [StoreCloser-TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/f3235915c4e94cb2b4eb1a98af2dfd87 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/f3235915c4e94cb2b4eb1a98af2dfd87 2024-12-07T15:16:35,849 DEBUG [StoreCloser-TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/f0f9bf5ef8f84566a416549159c55b3f to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/f0f9bf5ef8f84566a416549159c55b3f 2024-12-07T15:16:35,850 DEBUG [StoreCloser-TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/8360848629f44b259dedc3fa0e8b9183 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/8360848629f44b259dedc3fa0e8b9183 2024-12-07T15:16:35,851 DEBUG [StoreCloser-TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/8125d0e876ae445390a4cbf68416d7fc to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/8125d0e876ae445390a4cbf68416d7fc 2024-12-07T15:16:35,852 DEBUG [StoreCloser-TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/5a91b0641da04d5b9a1ee6718f29139f to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/5a91b0641da04d5b9a1ee6718f29139f 2024-12-07T15:16:35,852 DEBUG [StoreCloser-TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/c5ccf9755f644ac29d9cf2e061d5993a to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/c5ccf9755f644ac29d9cf2e061d5993a 2024-12-07T15:16:35,853 DEBUG [StoreCloser-TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/77f30e1d9c8b4bd28fbe1bd895182358 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/77f30e1d9c8b4bd28fbe1bd895182358 2024-12-07T15:16:35,854 DEBUG [StoreCloser-TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/b4cee46608704c61a33f2a7ae737111b to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/b4cee46608704c61a33f2a7ae737111b 2024-12-07T15:16:35,855 DEBUG [StoreCloser-TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/f10a3461d64f41b28fce5a1411358208 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/f10a3461d64f41b28fce5a1411358208 2024-12-07T15:16:35,855 DEBUG [StoreCloser-TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/363b4741cddf4ef18d59f14eec252152 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/363b4741cddf4ef18d59f14eec252152 2024-12-07T15:16:35,856 DEBUG [StoreCloser-TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/924683ad4a964bc0a951353c336b6672 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/924683ad4a964bc0a951353c336b6672 2024-12-07T15:16:35,860 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/recovered.edits/287.seqid, newMaxSeqId=287, maxSeqId=4 2024-12-07T15:16:35,860 INFO [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. 2024-12-07T15:16:35,860 DEBUG [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1676): Region close journal for 069dbda53fbb065f68f88519d6f94bc4: Waiting for close lock at 1733584594165Running coprocessor pre-close hooks at 1733584594165Disabling compacts and flushes for region at 1733584594165Disabling writes for close at 1733584594165Obtaining lock to block concurrent updates at 1733584594165Preparing flush snapshotting stores in 069dbda53fbb065f68f88519d6f94bc4 at 1733584594165Finished memstore snapshotting TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1733584594165Flushing stores of TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4. at 1733584594166 (+1 ms)Flushing 069dbda53fbb065f68f88519d6f94bc4/A: creating writer at 1733584594166Flushing mob file 069dbda53fbb065f68f88519d6f94bc4/A: appending metadata at 1733584594170 (+4 ms)Flushing mob file 069dbda53fbb065f68f88519d6f94bc4/A: closing flushed file at 1733584594170Flushing 069dbda53fbb065f68f88519d6f94bc4/A: appending metadata at 1733584594577 (+407 ms)Flushing 069dbda53fbb065f68f88519d6f94bc4/A: closing flushed file at 1733584594577Flushing 069dbda53fbb065f68f88519d6f94bc4/B: creating writer at 1733584594983 (+406 ms)Flushing 069dbda53fbb065f68f88519d6f94bc4/B: appending metadata at 1733584594985 (+2 ms)Flushing 069dbda53fbb065f68f88519d6f94bc4/B: closing flushed file at 1733584594985Flushing 069dbda53fbb065f68f88519d6f94bc4/C: creating writer at 1733584595391 (+406 ms)Flushing 069dbda53fbb065f68f88519d6f94bc4/C: appending metadata at 1733584595393 (+2 ms)Flushing 069dbda53fbb065f68f88519d6f94bc4/C: closing flushed file at 1733584595393Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4c51c592: reopening flushed file at 1733584595800 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1d2858d9: reopening flushed file at 1733584595803 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@c2b4c2c: reopening flushed file at 1733584595806 (+3 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 069dbda53fbb065f68f88519d6f94bc4 in 1644ms, sequenceid=284, compaction requested=true at 1733584595809 (+3 ms)Writing region close event to WAL at 1733584595857 (+48 ms)Running coprocessor post-close hooks at 1733584595860 (+3 ms)Closed at 1733584595860 2024-12-07T15:16:35,862 INFO [RS_CLOSE_REGION-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] handler.UnassignRegionHandler(157): Closed 069dbda53fbb065f68f88519d6f94bc4 2024-12-07T15:16:35,862 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=174 updating hbase:meta row=069dbda53fbb065f68f88519d6f94bc4, regionState=CLOSED 2024-12-07T15:16:35,863 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=175, ppid=174, state=RUNNABLE, hasLock=false; CloseRegionProcedure 069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 because future has completed 2024-12-07T15:16:35,865 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=175, resume processing ppid=174 2024-12-07T15:16:35,865 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=175, ppid=174, state=SUCCESS, hasLock=false; CloseRegionProcedure 069dbda53fbb065f68f88519d6f94bc4, server=d6797ebe9c52,46009,1733584418401 in 1.8510 sec 2024-12-07T15:16:35,867 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=174, resume processing ppid=173 2024-12-07T15:16:35,867 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=174, ppid=173, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=069dbda53fbb065f68f88519d6f94bc4, UNASSIGN in 1.8540 sec 2024-12-07T15:16:35,868 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=173, resume processing ppid=172 2024-12-07T15:16:35,869 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=173, ppid=172, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8570 sec 2024-12-07T15:16:35,869 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733584595869"}]},"ts":"1733584595869"} 2024-12-07T15:16:35,870 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-07T15:16:35,870 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-07T15:16:35,872 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=172, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8650 sec 2024-12-07T15:16:36,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-12-07T15:16:36,154 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-07T15:16:36,154 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-07T15:16:36,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] procedure2.ProcedureExecutor(1139): Stored pid=176, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-07T15:16:36,156 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=176, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-07T15:16:36,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-12-07T15:16:36,156 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=176, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-07T15:16:36,159 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4 2024-12-07T15:16:36,160 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A, FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B, FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C, FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/recovered.edits] 2024-12-07T15:16:36,162 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/0480f813b0f54c8a838c577f7ccbb7f7 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/0480f813b0f54c8a838c577f7ccbb7f7 2024-12-07T15:16:36,163 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/d7139f0d10bb4ee6aff55b614a67d795 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/d7139f0d10bb4ee6aff55b614a67d795 2024-12-07T15:16:36,164 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/e53766791bf04ad392e7f3cf311ed6ce to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/A/e53766791bf04ad392e7f3cf311ed6ce 2024-12-07T15:16:36,165 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/ae839e3e0fc2401f873851f7e5a4357a to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/ae839e3e0fc2401f873851f7e5a4357a 2024-12-07T15:16:36,166 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/af5978079cd84775af33fce837da49e6 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/af5978079cd84775af33fce837da49e6 2024-12-07T15:16:36,167 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/fe0db4ce3b2642eaa13e191f3e3a2b95 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/B/fe0db4ce3b2642eaa13e191f3e3a2b95 2024-12-07T15:16:36,168 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/446d4b95cd324914b62f6a3934fc3b0d to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/446d4b95cd324914b62f6a3934fc3b0d 2024-12-07T15:16:36,169 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/d42ab26cc9544c64897e6f1b32904502 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/d42ab26cc9544c64897e6f1b32904502 2024-12-07T15:16:36,169 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/fee6cd142fb4482788cb317d213c1b02 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/C/fee6cd142fb4482788cb317d213c1b02 2024-12-07T15:16:36,171 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/recovered.edits/287.seqid to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4/recovered.edits/287.seqid 2024-12-07T15:16:36,172 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/default/TestAcidGuarantees/069dbda53fbb065f68f88519d6f94bc4 2024-12-07T15:16:36,172 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-07T15:16:36,172 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-07T15:16:36,172 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-07T15:16:36,174 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412070f1e9b7d6a604ce8bfce9cbec5bc1630_069dbda53fbb065f68f88519d6f94bc4 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412070f1e9b7d6a604ce8bfce9cbec5bc1630_069dbda53fbb065f68f88519d6f94bc4 2024-12-07T15:16:36,175 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207299a0d5d243746d9ba3102b63368965f_069dbda53fbb065f68f88519d6f94bc4 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207299a0d5d243746d9ba3102b63368965f_069dbda53fbb065f68f88519d6f94bc4 2024-12-07T15:16:36,176 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120735dbfde0016c41c6ac8caae2249c66ed_069dbda53fbb065f68f88519d6f94bc4 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120735dbfde0016c41c6ac8caae2249c66ed_069dbda53fbb065f68f88519d6f94bc4 2024-12-07T15:16:36,177 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412073d09780822634b6982cbea9a3c9dff00_069dbda53fbb065f68f88519d6f94bc4 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412073d09780822634b6982cbea9a3c9dff00_069dbda53fbb065f68f88519d6f94bc4 2024-12-07T15:16:36,177 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412074fce63f886e745b6ada9b0967cd8585e_069dbda53fbb065f68f88519d6f94bc4 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412074fce63f886e745b6ada9b0967cd8585e_069dbda53fbb065f68f88519d6f94bc4 2024-12-07T15:16:36,178 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120753121cf77dde43ecb431f55203845870_069dbda53fbb065f68f88519d6f94bc4 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120753121cf77dde43ecb431f55203845870_069dbda53fbb065f68f88519d6f94bc4 2024-12-07T15:16:36,179 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412075d3289f000714ce3ad5581f029dc9931_069dbda53fbb065f68f88519d6f94bc4 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412075d3289f000714ce3ad5581f029dc9931_069dbda53fbb065f68f88519d6f94bc4 2024-12-07T15:16:36,180 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412075d9de84e3ff6491dbe135d9db1d0184c_069dbda53fbb065f68f88519d6f94bc4 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412075d9de84e3ff6491dbe135d9db1d0184c_069dbda53fbb065f68f88519d6f94bc4 2024-12-07T15:16:36,181 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412077a8ea6ec6fb048bf8c45652cb54a2ea3_069dbda53fbb065f68f88519d6f94bc4 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412077a8ea6ec6fb048bf8c45652cb54a2ea3_069dbda53fbb065f68f88519d6f94bc4 2024-12-07T15:16:36,181 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412077ffb7d497a244a8f8cdf06952994f5fd_069dbda53fbb065f68f88519d6f94bc4 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412077ffb7d497a244a8f8cdf06952994f5fd_069dbda53fbb065f68f88519d6f94bc4 2024-12-07T15:16:36,182 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120794cba35461c34335bd8cf779ec5b3210_069dbda53fbb065f68f88519d6f94bc4 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120794cba35461c34335bd8cf779ec5b3210_069dbda53fbb065f68f88519d6f94bc4 2024-12-07T15:16:36,183 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207a247122e626b4b9b80b86e5efda28ddb_069dbda53fbb065f68f88519d6f94bc4 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207a247122e626b4b9b80b86e5efda28ddb_069dbda53fbb065f68f88519d6f94bc4 2024-12-07T15:16:36,184 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207b7f114baf3394b718095824ad6850079_069dbda53fbb065f68f88519d6f94bc4 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207b7f114baf3394b718095824ad6850079_069dbda53fbb065f68f88519d6f94bc4 2024-12-07T15:16:36,185 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207bf89dec20d7f4626ae0290b83100165d_069dbda53fbb065f68f88519d6f94bc4 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207bf89dec20d7f4626ae0290b83100165d_069dbda53fbb065f68f88519d6f94bc4 2024-12-07T15:16:36,186 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207ca169a7c874b43dfaea588bc3abcf307_069dbda53fbb065f68f88519d6f94bc4 to hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207ca169a7c874b43dfaea588bc3abcf307_069dbda53fbb065f68f88519d6f94bc4 2024-12-07T15:16:36,186 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-07T15:16:36,188 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=176, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-07T15:16:36,190 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-07T15:16:36,191 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-07T15:16:36,192 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=176, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-07T15:16:36,192 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-07T15:16:36,192 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733584596192"}]},"ts":"9223372036854775807"} 2024-12-07T15:16:36,194 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-07T15:16:36,194 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 069dbda53fbb065f68f88519d6f94bc4, NAME => 'TestAcidGuarantees,,1733584562452.069dbda53fbb065f68f88519d6f94bc4.', STARTKEY => '', ENDKEY => ''}] 2024-12-07T15:16:36,194 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-07T15:16:36,194 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733584596194"}]},"ts":"9223372036854775807"} 2024-12-07T15:16:36,195 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-07T15:16:36,195 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=176, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-07T15:16:36,196 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=176, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 41 msec 2024-12-07T15:16:36,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-12-07T15:16:36,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-07T15:16:36,264 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-07T15:16:36,273 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=248 (was 250), OpenFileDescriptor=441 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=337 (was 396), ProcessCount=11 (was 11), AvailableMemoryMB=6700 (was 6746) 2024-12-07T15:16:36,273 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-07T15:16:36,273 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-07T15:16:36,273 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-07T15:16:36,273 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:16:36,273 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:16:36,273 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T15:16:36,273 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-07T15:16:36,273 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=2127823695, stopped=false 2024-12-07T15:16:36,274 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=d6797ebe9c52,36915,1733584417704 2024-12-07T15:16:36,275 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46009-0x1018d7cdad70001, quorum=127.0.0.1:52406, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-07T15:16:36,275 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36915-0x1018d7cdad70000, quorum=127.0.0.1:52406, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-07T15:16:36,275 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46009-0x1018d7cdad70001, quorum=127.0.0.1:52406, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T15:16:36,275 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36915-0x1018d7cdad70000, quorum=127.0.0.1:52406, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T15:16:36,275 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T15:16:36,276 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-07T15:16:36,276 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-07T15:16:36,276 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:16:36,276 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:46009-0x1018d7cdad70001, quorum=127.0.0.1:52406, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-07T15:16:36,276 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:36915-0x1018d7cdad70000, quorum=127.0.0.1:52406, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-07T15:16:36,277 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'd6797ebe9c52,46009,1733584418401' ***** 2024-12-07T15:16:36,277 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-07T15:16:36,277 INFO [RS:0;d6797ebe9c52:46009 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-07T15:16:36,277 INFO [RS:0;d6797ebe9c52:46009 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-07T15:16:36,277 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-07T15:16:36,277 INFO [RS:0;d6797ebe9c52:46009 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-07T15:16:36,277 INFO [RS:0;d6797ebe9c52:46009 {}] regionserver.HRegionServer(959): stopping server d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:36,278 INFO [RS:0;d6797ebe9c52:46009 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-07T15:16:36,278 INFO [RS:0;d6797ebe9c52:46009 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;d6797ebe9c52:46009. 2024-12-07T15:16:36,278 DEBUG [RS:0;d6797ebe9c52:46009 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-07T15:16:36,278 DEBUG [RS:0;d6797ebe9c52:46009 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:16:36,278 INFO [RS:0;d6797ebe9c52:46009 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-07T15:16:36,278 INFO [RS:0;d6797ebe9c52:46009 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-07T15:16:36,278 INFO [RS:0;d6797ebe9c52:46009 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-07T15:16:36,278 INFO [RS:0;d6797ebe9c52:46009 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-07T15:16:36,278 INFO [RS:0;d6797ebe9c52:46009 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-07T15:16:36,278 DEBUG [RS:0;d6797ebe9c52:46009 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-12-07T15:16:36,279 DEBUG [RS:0;d6797ebe9c52:46009 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-07T15:16:36,279 DEBUG [RS_CLOSE_META-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-07T15:16:36,279 INFO [RS_CLOSE_META-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-07T15:16:36,279 DEBUG [RS_CLOSE_META-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-07T15:16:36,279 DEBUG [RS_CLOSE_META-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-07T15:16:36,279 DEBUG [RS_CLOSE_META-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-07T15:16:36,279 INFO [RS_CLOSE_META-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-12-07T15:16:36,304 DEBUG [RS_CLOSE_META-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/hbase/meta/1588230740/.tmp/info/407b579733d94876914bb1f059531c69 is 95, key is TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2./info:/1733584444670/DeleteFamily/seqid=0 2024-12-07T15:16:36,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742445_1621 (size=6514) 2024-12-07T15:16:36,366 INFO [regionserver/d6797ebe9c52:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-07T15:16:36,479 DEBUG [RS:0;d6797ebe9c52:46009 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-07T15:16:36,679 DEBUG [RS:0;d6797ebe9c52:46009 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-07T15:16:36,708 INFO [RS_CLOSE_META-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/hbase/meta/1588230740/.tmp/info/407b579733d94876914bb1f059531c69 2024-12-07T15:16:36,726 DEBUG [RS_CLOSE_META-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/hbase/meta/1588230740/.tmp/ns/1ce0251649ff40079da6fbf417d95896 is 93, key is TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2./ns:/1733584444610/DeleteFamily/seqid=0 2024-12-07T15:16:36,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742446_1622 (size=5906) 2024-12-07T15:16:36,776 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-07T15:16:36,879 DEBUG [RS:0;d6797ebe9c52:46009 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-07T15:16:37,079 DEBUG [RS:0;d6797ebe9c52:46009 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-07T15:16:37,124 INFO [regionserver/d6797ebe9c52:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-07T15:16:37,124 INFO [regionserver/d6797ebe9c52:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-07T15:16:37,129 INFO [RS_CLOSE_META-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/hbase/meta/1588230740/.tmp/ns/1ce0251649ff40079da6fbf417d95896 2024-12-07T15:16:37,146 DEBUG [RS_CLOSE_META-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/hbase/meta/1588230740/.tmp/rep_barrier/8eb02b9bfcc64012a518b9a29a651ddb is 102, key is TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2./rep_barrier:/1733584444610/DeleteFamily/seqid=0 2024-12-07T15:16:37,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742447_1623 (size=6025) 2024-12-07T15:16:37,280 DEBUG [RS:0;d6797ebe9c52:46009 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-07T15:16:37,480 DEBUG [RS:0;d6797ebe9c52:46009 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-07T15:16:37,550 INFO [RS_CLOSE_META-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/hbase/meta/1588230740/.tmp/rep_barrier/8eb02b9bfcc64012a518b9a29a651ddb 2024-12-07T15:16:37,569 DEBUG [RS_CLOSE_META-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/hbase/meta/1588230740/.tmp/table/af4081d64aa74bddb85095d35ea2f899 is 96, key is TestAcidGuarantees,,1733584421246.53401f0681ce510d042c1599ea9bc7e2./table:/1733584444610/DeleteFamily/seqid=0 2024-12-07T15:16:37,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742448_1624 (size=5936) 2024-12-07T15:16:37,680 DEBUG [RS:0;d6797ebe9c52:46009 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-07T15:16:37,880 DEBUG [RS:0;d6797ebe9c52:46009 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-07T15:16:37,972 INFO [RS_CLOSE_META-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/hbase/meta/1588230740/.tmp/table/af4081d64aa74bddb85095d35ea2f899 2024-12-07T15:16:37,976 DEBUG [RS_CLOSE_META-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/hbase/meta/1588230740/.tmp/info/407b579733d94876914bb1f059531c69 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/hbase/meta/1588230740/info/407b579733d94876914bb1f059531c69 2024-12-07T15:16:37,978 INFO [RS_CLOSE_META-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/hbase/meta/1588230740/info/407b579733d94876914bb1f059531c69, entries=12, sequenceid=90, filesize=6.4 K 2024-12-07T15:16:37,979 DEBUG [RS_CLOSE_META-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/hbase/meta/1588230740/.tmp/ns/1ce0251649ff40079da6fbf417d95896 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/hbase/meta/1588230740/ns/1ce0251649ff40079da6fbf417d95896 2024-12-07T15:16:37,982 INFO [RS_CLOSE_META-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/hbase/meta/1588230740/ns/1ce0251649ff40079da6fbf417d95896, entries=8, sequenceid=90, filesize=5.8 K 2024-12-07T15:16:37,982 DEBUG [RS_CLOSE_META-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/hbase/meta/1588230740/.tmp/rep_barrier/8eb02b9bfcc64012a518b9a29a651ddb as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/hbase/meta/1588230740/rep_barrier/8eb02b9bfcc64012a518b9a29a651ddb 2024-12-07T15:16:37,984 INFO [RS_CLOSE_META-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/hbase/meta/1588230740/rep_barrier/8eb02b9bfcc64012a518b9a29a651ddb, entries=6, sequenceid=90, filesize=5.9 K 2024-12-07T15:16:37,985 DEBUG [RS_CLOSE_META-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/hbase/meta/1588230740/.tmp/table/af4081d64aa74bddb85095d35ea2f899 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/hbase/meta/1588230740/table/af4081d64aa74bddb85095d35ea2f899 2024-12-07T15:16:37,987 INFO [RS_CLOSE_META-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/hbase/meta/1588230740/table/af4081d64aa74bddb85095d35ea2f899, entries=7, sequenceid=90, filesize=5.8 K 2024-12-07T15:16:37,988 INFO [RS_CLOSE_META-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1709ms, sequenceid=90, compaction requested=false 2024-12-07T15:16:37,992 DEBUG [RS_CLOSE_META-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-12-07T15:16:37,993 DEBUG [RS_CLOSE_META-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-07T15:16:37,993 INFO [RS_CLOSE_META-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-07T15:16:37,993 DEBUG [RS_CLOSE_META-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733584596278Running coprocessor pre-close hooks at 1733584596279 (+1 ms)Disabling compacts and flushes for region at 1733584596279Disabling writes for close at 1733584596279Obtaining lock to block concurrent updates at 1733584596279Preparing flush snapshotting stores in 1588230740 at 1733584596279Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1733584596280 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733584596281 (+1 ms)Flushing 1588230740/info: creating writer at 1733584596281Flushing 1588230740/info: appending metadata at 1733584596303 (+22 ms)Flushing 1588230740/info: closing flushed file at 1733584596304 (+1 ms)Flushing 1588230740/ns: creating writer at 1733584596711 (+407 ms)Flushing 1588230740/ns: appending metadata at 1733584596725 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1733584596726 (+1 ms)Flushing 1588230740/rep_barrier: creating writer at 1733584597132 (+406 ms)Flushing 1588230740/rep_barrier: appending metadata at 1733584597146 (+14 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1733584597146Flushing 1588230740/table: creating writer at 1733584597553 (+407 ms)Flushing 1588230740/table: appending metadata at 1733584597568 (+15 ms)Flushing 1588230740/table: closing flushed file at 1733584597568Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7135a47d: reopening flushed file at 1733584597975 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@17f5af8a: reopening flushed file at 1733584597979 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7a74a09: reopening flushed file at 1733584597982 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1173ee69: reopening flushed file at 1733584597985 (+3 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1709ms, sequenceid=90, compaction requested=false at 1733584597988 (+3 ms)Writing region close event to WAL at 1733584597990 (+2 ms)Running coprocessor post-close hooks at 1733584597992 (+2 ms)Closed at 1733584597993 (+1 ms) 2024-12-07T15:16:37,993 DEBUG [RS_CLOSE_META-regionserver/d6797ebe9c52:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-07T15:16:38,081 INFO [RS:0;d6797ebe9c52:46009 {}] regionserver.HRegionServer(976): stopping server d6797ebe9c52,46009,1733584418401; all regions closed. 2024-12-07T15:16:38,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741834_1010 (size=25344) 2024-12-07T15:16:38,087 DEBUG [RS:0;d6797ebe9c52:46009 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/oldWALs 2024-12-07T15:16:38,087 INFO [RS:0;d6797ebe9c52:46009 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL d6797ebe9c52%2C46009%2C1733584418401.meta:.meta(num 1733584420661) 2024-12-07T15:16:38,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741833_1009 (size=18112055) 2024-12-07T15:16:38,090 DEBUG [RS:0;d6797ebe9c52:46009 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/oldWALs 2024-12-07T15:16:38,090 INFO [RS:0;d6797ebe9c52:46009 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL d6797ebe9c52%2C46009%2C1733584418401:(num 1733584420290) 2024-12-07T15:16:38,090 DEBUG [RS:0;d6797ebe9c52:46009 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T15:16:38,090 INFO [RS:0;d6797ebe9c52:46009 {}] regionserver.LeaseManager(133): Closed leases 2024-12-07T15:16:38,090 INFO [RS:0;d6797ebe9c52:46009 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-07T15:16:38,090 INFO [RS:0;d6797ebe9c52:46009 {}] hbase.ChoreService(370): Chore service for: regionserver/d6797ebe9c52:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-07T15:16:38,090 INFO [RS:0;d6797ebe9c52:46009 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-07T15:16:38,090 INFO [regionserver/d6797ebe9c52:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-07T15:16:38,091 INFO [RS:0;d6797ebe9c52:46009 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46009 2024-12-07T15:16:38,095 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46009-0x1018d7cdad70001, quorum=127.0.0.1:52406, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/d6797ebe9c52,46009,1733584418401 2024-12-07T15:16:38,095 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36915-0x1018d7cdad70000, quorum=127.0.0.1:52406, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-07T15:16:38,095 INFO [RS:0;d6797ebe9c52:46009 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-07T15:16:38,096 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [d6797ebe9c52,46009,1733584418401] 2024-12-07T15:16:38,098 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/d6797ebe9c52,46009,1733584418401 already deleted, retry=false 2024-12-07T15:16:38,098 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; d6797ebe9c52,46009,1733584418401 expired; onlineServers=0 2024-12-07T15:16:38,098 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'd6797ebe9c52,36915,1733584417704' ***** 2024-12-07T15:16:38,098 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-07T15:16:38,098 INFO [M:0;d6797ebe9c52:36915 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-07T15:16:38,098 INFO [M:0;d6797ebe9c52:36915 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-07T15:16:38,098 DEBUG [M:0;d6797ebe9c52:36915 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-07T15:16:38,098 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-07T15:16:38,098 DEBUG [M:0;d6797ebe9c52:36915 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-07T15:16:38,099 DEBUG [master/d6797ebe9c52:0:becomeActiveMaster-HFileCleaner.small.0-1733584419898 {}] cleaner.HFileCleaner(306): Exit Thread[master/d6797ebe9c52:0:becomeActiveMaster-HFileCleaner.small.0-1733584419898,5,FailOnTimeoutGroup] 2024-12-07T15:16:38,099 DEBUG [master/d6797ebe9c52:0:becomeActiveMaster-HFileCleaner.large.0-1733584419894 {}] cleaner.HFileCleaner(306): Exit Thread[master/d6797ebe9c52:0:becomeActiveMaster-HFileCleaner.large.0-1733584419894,5,FailOnTimeoutGroup] 2024-12-07T15:16:38,099 INFO [M:0;d6797ebe9c52:36915 {}] hbase.ChoreService(370): Chore service for: master/d6797ebe9c52:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-07T15:16:38,099 INFO [M:0;d6797ebe9c52:36915 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-07T15:16:38,099 DEBUG [M:0;d6797ebe9c52:36915 {}] master.HMaster(1795): Stopping service threads 2024-12-07T15:16:38,099 INFO [M:0;d6797ebe9c52:36915 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-07T15:16:38,099 INFO [M:0;d6797ebe9c52:36915 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T15:16:38,099 ERROR [M:0;d6797ebe9c52:36915 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[IPC Client (1102635514) connection to localhost/127.0.0.1:36357 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:36357,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-12-07T15:16:38,100 INFO [M:0;d6797ebe9c52:36915 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-07T15:16:38,100 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-07T15:16:38,101 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36915-0x1018d7cdad70000, quorum=127.0.0.1:52406, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-07T15:16:38,101 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36915-0x1018d7cdad70000, quorum=127.0.0.1:52406, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T15:16:38,102 DEBUG [M:0;d6797ebe9c52:36915 {}] zookeeper.ZKUtil(347): master:36915-0x1018d7cdad70000, quorum=127.0.0.1:52406, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-07T15:16:38,102 WARN [M:0;d6797ebe9c52:36915 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-07T15:16:38,102 INFO [M:0;d6797ebe9c52:36915 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/.lastflushedseqids 2024-12-07T15:16:38,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742449_1625 (size=72) 2024-12-07T15:16:38,122 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-07T15:16:38,122 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-07T15:16:38,122 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-07T15:16:38,197 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46009-0x1018d7cdad70001, quorum=127.0.0.1:52406, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-07T15:16:38,197 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46009-0x1018d7cdad70001, quorum=127.0.0.1:52406, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-07T15:16:38,197 INFO [RS:0;d6797ebe9c52:46009 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-07T15:16:38,198 INFO [RS:0;d6797ebe9c52:46009 {}] regionserver.HRegionServer(1031): Exiting; stopping=d6797ebe9c52,46009,1733584418401; zookeeper connection closed. 2024-12-07T15:16:38,198 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@5064fa55 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@5064fa55 2024-12-07T15:16:38,198 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-07T15:16:38,512 INFO [M:0;d6797ebe9c52:36915 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-07T15:16:38,512 INFO [M:0;d6797ebe9c52:36915 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-07T15:16:38,512 DEBUG [M:0;d6797ebe9c52:36915 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-07T15:16:38,512 INFO [M:0;d6797ebe9c52:36915 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-07T15:16:38,512 DEBUG [M:0;d6797ebe9c52:36915 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-07T15:16:38,512 DEBUG [M:0;d6797ebe9c52:36915 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-07T15:16:38,512 DEBUG [M:0;d6797ebe9c52:36915 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-07T15:16:38,512 INFO [M:0;d6797ebe9c52:36915 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=791.38 KB heapSize=970.59 KB 2024-12-07T15:16:38,528 DEBUG [M:0;d6797ebe9c52:36915 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/60bd9315af4e41f38400ad1f4e040c28 is 82, key is hbase:meta,,1/info:regioninfo/1733584420788/Put/seqid=0 2024-12-07T15:16:38,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742450_1626 (size=5672) 2024-12-07T15:16:38,931 INFO [M:0;d6797ebe9c52:36915 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2212 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/60bd9315af4e41f38400ad1f4e040c28 2024-12-07T15:16:38,952 DEBUG [M:0;d6797ebe9c52:36915 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b0b138c2edef48919c6141a4d6a5cf8e is 2479, key is \x00\x00\x00\x00\x00\x00\x00`/proc:d/1733584505840/Put/seqid=0 2024-12-07T15:16:38,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742451_1627 (size=46398) 2024-12-07T15:16:39,355 INFO [M:0;d6797ebe9c52:36915 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=790.82 KB at sequenceid=2212 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b0b138c2edef48919c6141a4d6a5cf8e 2024-12-07T15:16:39,358 INFO [M:0;d6797ebe9c52:36915 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for b0b138c2edef48919c6141a4d6a5cf8e 2024-12-07T15:16:39,373 DEBUG [M:0;d6797ebe9c52:36915 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/c67ad9ad95604b0f83f82bfbeba3ae62 is 69, key is d6797ebe9c52,46009,1733584418401/rs:state/1733584420047/Put/seqid=0 2024-12-07T15:16:39,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073742452_1628 (size=5156) 2024-12-07T15:16:39,777 INFO [M:0;d6797ebe9c52:36915 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2212 (bloomFilter=true), to=hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/c67ad9ad95604b0f83f82bfbeba3ae62 2024-12-07T15:16:39,780 DEBUG [M:0;d6797ebe9c52:36915 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/60bd9315af4e41f38400ad1f4e040c28 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/60bd9315af4e41f38400ad1f4e040c28 2024-12-07T15:16:39,782 INFO [M:0;d6797ebe9c52:36915 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/60bd9315af4e41f38400ad1f4e040c28, entries=8, sequenceid=2212, filesize=5.5 K 2024-12-07T15:16:39,783 DEBUG [M:0;d6797ebe9c52:36915 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b0b138c2edef48919c6141a4d6a5cf8e as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/b0b138c2edef48919c6141a4d6a5cf8e 2024-12-07T15:16:39,785 INFO [M:0;d6797ebe9c52:36915 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for b0b138c2edef48919c6141a4d6a5cf8e 2024-12-07T15:16:39,785 INFO [M:0;d6797ebe9c52:36915 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/b0b138c2edef48919c6141a4d6a5cf8e, entries=176, sequenceid=2212, filesize=45.3 K 2024-12-07T15:16:39,785 DEBUG [M:0;d6797ebe9c52:36915 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/c67ad9ad95604b0f83f82bfbeba3ae62 as hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/c67ad9ad95604b0f83f82bfbeba3ae62 2024-12-07T15:16:39,788 INFO [M:0;d6797ebe9c52:36915 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36357/user/jenkins/test-data/b49fabc3-1884-cc54-612b-74c81f03a6bd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/c67ad9ad95604b0f83f82bfbeba3ae62, entries=1, sequenceid=2212, filesize=5.0 K 2024-12-07T15:16:39,789 INFO [M:0;d6797ebe9c52:36915 {}] regionserver.HRegion(3140): Finished flush of dataSize ~791.38 KB/810370, heapSize ~970.30 KB/993584, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1277ms, sequenceid=2212, compaction requested=false 2024-12-07T15:16:39,790 INFO [M:0;d6797ebe9c52:36915 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-07T15:16:39,790 DEBUG [M:0;d6797ebe9c52:36915 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733584598512Disabling compacts and flushes for region at 1733584598512Disabling writes for close at 1733584598512Obtaining lock to block concurrent updates at 1733584598512Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733584598512Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=810370, getHeapSize=993824, getOffHeapSize=0, getCellsCount=2413 at 1733584598513 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733584598513Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733584598513Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733584598527 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733584598528 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733584598935 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733584598951 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733584598951Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733584599359 (+408 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733584599373 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733584599373Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6d0e82e0: reopening flushed file at 1733584599779 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@788e2e2c: reopening flushed file at 1733584599782 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@74f81435: reopening flushed file at 1733584599785 (+3 ms)Finished flush of dataSize ~791.38 KB/810370, heapSize ~970.30 KB/993584, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1277ms, sequenceid=2212, compaction requested=false at 1733584599789 (+4 ms)Writing region close event to WAL at 1733584599790 (+1 ms)Closed at 1733584599790 2024-12-07T15:16:39,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43901 is added to blk_1073741830_1006 (size=955373) 2024-12-07T15:16:39,793 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-07T15:16:39,793 INFO [M:0;d6797ebe9c52:36915 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-07T15:16:39,793 INFO [M:0;d6797ebe9c52:36915 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36915 2024-12-07T15:16:39,793 INFO [M:0;d6797ebe9c52:36915 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-07T15:16:39,895 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36915-0x1018d7cdad70000, quorum=127.0.0.1:52406, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-07T15:16:39,895 INFO [M:0;d6797ebe9c52:36915 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-07T15:16:39,895 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36915-0x1018d7cdad70000, quorum=127.0.0.1:52406, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-07T15:16:39,900 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-07T15:16:39,902 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-07T15:16:39,902 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-07T15:16:39,902 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-07T15:16:39,902 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5975031d-e054-e0fa-13a8-f8e344c3a20d/hadoop.log.dir/,STOPPED} 2024-12-07T15:16:39,905 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-07T15:16:39,905 WARN [BP-848825693-172.17.0.2-1733584414802 heartbeating to localhost/127.0.0.1:36357 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-07T15:16:39,905 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-07T15:16:39,905 WARN [BP-848825693-172.17.0.2-1733584414802 heartbeating to localhost/127.0.0.1:36357 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-848825693-172.17.0.2-1733584414802 (Datanode Uuid 4d6fefba-0d69-4d1c-92e9-8f4bfc1f4da9) service to localhost/127.0.0.1:36357 2024-12-07T15:16:39,907 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5975031d-e054-e0fa-13a8-f8e344c3a20d/cluster_8a1d19e0-6a6c-4b1f-8318-b9e148680dc0/data/data1/current/BP-848825693-172.17.0.2-1733584414802 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-07T15:16:39,907 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5975031d-e054-e0fa-13a8-f8e344c3a20d/cluster_8a1d19e0-6a6c-4b1f-8318-b9e148680dc0/data/data2/current/BP-848825693-172.17.0.2-1733584414802 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-07T15:16:39,908 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-07T15:16:39,915 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-07T15:16:39,916 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-07T15:16:39,916 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-07T15:16:39,916 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-07T15:16:39,916 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5975031d-e054-e0fa-13a8-f8e344c3a20d/hadoop.log.dir/,STOPPED} 2024-12-07T15:16:39,932 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-07T15:16:40,072 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down